rb: reading report...done. rb: reading report...done. INFO REPORT <0.53.0> 2011-07-13 23:49:36 =============================================================================== nonode@nohost:<0.53.0>:log_os_info:25: OS type: {unix,linux} Version: {2,6,18} Runtime info: [{otp_release,"R14B02"}, {erl_version,"5.8.3"}, {erl_version_long, "Erlang R14B02 (erts-5.8.3) [source] [64-bit] [rq:1] [async-threads:16] [hipe] [kernel-poll:false]\n"}, {system_arch_raw,"x86_64-unknown-linux-gnu"}, {system_arch,"x86_64-unknown-linux-gnu"}, {localtime,{{2011,7,13},{23,49,36}}}, {memory, [{total,14619760}, {processes,1395760}, {processes_used,1380848}, {system,13224000}, {atom,448857}, {atom_used,419150}, {binary,65328}, {code,3299798}, {ets,267344}]}, {loaded, [ns_info,log_os_info,misc,ns_log_mf_h, ns_server_cluster_sup,ns_server,cpu_sup,memsup,disksup, os_mon,io_lib_pretty,io_lib_format,io_lib,io,sasl_report, release_handler,calendar,overload,alarm_handler, sasl_report_tty_h,sasl,ns_bootstrap,file_io_server, orddict,erl_eval,c,error_logger_tty_h,queue,kernel_config, user,user_sup,supervisor_bridge,standard_error,ram_file, file,beam_lib,unicode,binary,ets,gb_sets, hipe_unified_loader,packages,code_server,code,file_server, net_kernel,global_group,erl_distribution,filename, inet_gethost_native,inet_parse,inet,inet_udp,os, inet_config,inet_db,global,gb_trees,rpc,supervisor,kernel, application_master,sys,application,gen_server,erl_parse, proplists,erl_scan,lists,application_controller,proc_lib, gen,gen_event,error_logger,heart,error_handler,erlang, erl_prim_loader,prim_zip,zlib,prim_file,prim_inet,init, otp_ring0]}, {applications, [{kernel,"ERTS CXC 138 10","2.14.3"}, {stdlib,"ERTS CXC 138 10","1.17.3"}, {os_mon,"CPO CXC 138 46","2.2.5"}, {ns_server,"Membase server","1.7.1r-42-gd713cca"}, {sasl,"SASL CXC 138 11","2.1.9.3"}]}, {pre_loaded, [erlang,erl_prim_loader,prim_zip,zlib,prim_file,prim_inet, init,otp_ring0]}, {process_count,43}, {node,nonode@nohost}, {nodes,[]}, {registered, [rex,release_handler,inet_db,kernel_sup, ns_server_cluster_sup,code_server,global_name_server, overload,file_server_2,application_controller, alarm_handler,os_mon_sup,standard_error,init,cpu_sup, global_group,erl_prim_loader,sasl_sup,memsup,disksup, standard_error_sup,user,sasl_safe_sup,error_logger, kernel_safe_sup]}, {cookie,nocookie}, {wordsize,8}, {wall_clock,0}] INFO REPORT <0.53.0> 2011-07-13 23:49:36 =============================================================================== nonode@nohost:<0.53.0>:log_os_info:27: Manifest: ["bucket_engine 1.7.0-0-g721dff0 Linux-x86_64", "ep-engine 1.7.0-33-ged2a54a Linux-x86_64", "geocouch couchbase_1.1_geo-0-gd09068e Linux-x86_64", "icu4c 2635939 Linux-x86_64","libconflate 1.7.0-0-gfee8f94 Linux-x86_64", "libmemcached 2cd40dc Linux-x86_64", "libvbucket 1.7.0-0-g4bd0aba Linux-x86_64", "manifest 1.7.0-13-gc78c994 Linux-x86_64", "membase-cli 1.7.0-1-g45a4232 Linux-x86_64", "membase-server 1.7.1r-42-gd713cca Linux-x86_64", "membasex 1.7.0-0-gf85471c Linux-x86_64", "memcached 1.4.4-461-gf99c147 Linux-x86_64", "memcachetest 0.8.3-0-g88ae3b3 Linux-x86_64", "moxi 1.7.0-4-g6a26e91 Linux-x86_64", "ns_server 1.7.0-74-gd6663ae Linux-x86_64", "otp OTP_R14B02-0-g91b2e57 Linux-x86_64", "portsigar 1.7.0-0-ga191e6c Linux-x86_64", "sigar sigar-1.6.4-406-ge1dcf32 Linux-x86_64", "spidermonkey a3c48c1 Linux-x86_64","tlm 1.7.0-8-gce9a2e8 Linux-x86_64", "vbucketmigrator 1.7.0-0-g0fdc96c Linux-x86_64"] PROGRESS REPORT <0.53.0> 2011-07-13 23:49:36 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<0.55.0>}, {name,timeout_diag_logger}, {mfargs,{timeout_diag_logger,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <0.56.0> 2011-07-13 23:49:36 =============================================================================== reading ip config from "/opt/membase/var/lib/membase/ip" ERROR REPORT <0.56.0> 2011-07-13 23:49:36 =============================================================================== Got error:einval. Ignoring bad address:[] INFO REPORT <0.56.0> 2011-07-13 23:49:36 =============================================================================== nonode@nohost:<0.56.0>:dist_manager:105: Attempting to bring up net_kernel with name 'ns_1@127.0.0.1' PROGRESS REPORT <0.57.0> 2011-07-13 23:49:36 =============================================================================== supervisor {local,net_sup} started [{pid,<0.58.0>}, {name,erl_epmd}, {mfargs,{erl_epmd,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] PROGRESS REPORT <0.57.0> 2011-07-13 23:49:36 =============================================================================== supervisor {local,net_sup} started [{pid,<0.59.0>}, {name,auth}, {mfargs,{auth,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] PROGRESS REPORT <6093.57.0> 2011-07-13 23:49:36 =============================================================================== supervisor {local,net_sup} started [{pid,<6093.60.0>}, {name,net_kernel}, {mfargs,{net_kernel,start_link,[['ns_1@127.0.0.1',longnames]]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] PROGRESS REPORT <6093.11.0> 2011-07-13 23:49:36 =============================================================================== supervisor {local,kernel_sup} started [{pid,<6093.57.0>}, {name,net_sup_dynamic}, {mfargs,{erl_distribution,start_link, [['ns_1@127.0.0.1',longnames]]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,supervisor}] PROGRESS REPORT <6093.53.0> 2011-07-13 23:49:36 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<6093.56.0>}, {name,dist_manager}, {mfargs,{dist_manager,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6093.53.0> 2011-07-13 23:49:36 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<6093.63.0>}, {name,ns_cluster}, {mfargs,{ns_cluster,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <6093.64.0> 2011-07-13 23:49:36 =============================================================================== supervisor {local,mb_mnesia_sup} started [{pid,<6093.65.0>}, {name,mb_mnesia_events}, {mfargs,{gen_event,start_link,[{local,mb_mnesia_events}]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <6093.71.0> 2011-07-13 23:49:36 =============================================================================== supervisor {local,mnesia_sup} started [{pid,<6093.72.0>}, {name,mnesia_event}, {mfargs,{mnesia_sup,start_event,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] PROGRESS REPORT <6093.73.0> 2011-07-13 23:49:36 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<6093.74.0>}, {name,mnesia_monitor}, {mfargs,{mnesia_monitor,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <6093.73.0> 2011-07-13 23:49:36 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<6093.75.0>}, {name,mnesia_subscr}, {mfargs,{mnesia_subscr,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <6093.73.0> 2011-07-13 23:49:36 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<6093.76.0>}, {name,mnesia_locker}, {mfargs,{mnesia_locker,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <6093.73.0> 2011-07-13 23:49:36 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<6093.77.0>}, {name,mnesia_recover}, {mfargs,{mnesia_recover,start,[]}}, {restart_type,permanent}, {shutdown,180000}, {child_type,worker}] PROGRESS REPORT <6093.26.0> 2011-07-13 23:49:36 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<6093.79.0>}, {name,timer_server}, {mfargs,{timer,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6093.73.0> 2011-07-13 23:49:36 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<6093.78.0>}, {name,mnesia_tm}, {mfargs,{mnesia_tm,start,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] PROGRESS REPORT <6093.73.0> 2011-07-13 23:49:36 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<6093.80.0>}, {name,mnesia_checkpoint_sup}, {mfargs,{mnesia_checkpoint_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <6093.73.0> 2011-07-13 23:49:36 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<6093.81.0>}, {name,mnesia_snmp_sup}, {mfargs,{mnesia_snmp_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <6093.73.0> 2011-07-13 23:49:36 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<6093.82.0>}, {name,mnesia_controller}, {mfargs,{mnesia_controller,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <6093.73.0> 2011-07-13 23:49:36 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<6093.83.0>}, {name,mnesia_late_loader}, {mfargs,{mnesia_late_loader,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <6093.71.0> 2011-07-13 23:49:36 =============================================================================== supervisor {local,mnesia_sup} started [{pid,<6093.73.0>}, {name,mnesia_kernel_sup}, {mfargs,{mnesia_kernel_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <6093.7.0> 2011-07-13 23:49:36 =============================================================================== application mnesia started_at 'ns_1@127.0.0.1' PROGRESS REPORT <6093.26.0> 2011-07-13 23:49:36 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<6093.87.0>}, {name,disk_log_sup}, {mfargs,{disk_log_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,supervisor}] PROGRESS REPORT <6093.26.0> 2011-07-13 23:49:36 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<6093.88.0>}, {name,disk_log_server}, {mfargs,{disk_log_server,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] PROGRESS REPORT <6093.26.0> 2011-07-13 23:49:38 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<6093.91.0>}, {name,dets_sup}, {mfargs,{dets_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,supervisor}] PROGRESS REPORT <6093.26.0> 2011-07-13 23:49:38 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<6093.92.0>}, {name,dets}, {mfargs,{dets_server,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] INFO REPORT <6093.66.0> 2011-07-13 23:49:38 =============================================================================== ns_1@127.0.0.1:<6093.66.0>:mb_mnesia:409: Committed schema to disk. INFO REPORT <6093.66.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.66.0>:mb_mnesia:245: Current config: [{access_module,mnesia}, {auto_repair,true}, {backup_module,mnesia_backup}, {checkpoints,[]}, {db_nodes,['ns_1@127.0.0.1']}, {debug,verbose}, {directory,"/opt/membase/var/lib/membase/mnesia"}, {dump_log_load_regulation,false}, {dump_log_time_threshold,180000}, {dump_log_update_in_place,true}, {dump_log_write_threshold,1000}, {embedded_mnemosyne,false}, {event_module,mnesia_event}, {extra_db_nodes,[]}, {fallback_activated,false}, {held_locks,[]}, {ignore_fallback_at_startup,false}, {fallback_error_function,{mnesia,lkill}}, {is_running,yes}, {local_tables,[local_config,cluster,schema]}, {lock_queue,[]}, {log_version,"4.3"}, {master_node_tables,[]}, {max_wait_for_decision,10000}, {protocol_version,{7,6}}, {running_db_nodes,['ns_1@127.0.0.1']}, {schema_location,opt_disc}, {schema_version,{3,0}}, {subscribers,[<6093.72.0>,<6093.66.0>]}, {tables,[local_config,cluster,schema]}, {transaction_commits,5}, {transaction_failures,0}, {transaction_log_writes,3}, {transaction_restarts,0}, {transactions,[]}, {use_dir,true}, {core_dir,false}, {no_table_loaders,2}, {dc_dump_limit,4}, {send_compressed,0}, {version,"4.4.17"}] Peers: ['ns_1@127.0.0.1'] INFO REPORT <6093.66.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.66.0>:mb_mnesia:199: Mnesia table event: {write,{schema,schema, [{name,schema}, {type,set}, {ram_copies,[]}, {disc_copies,['ns_1@127.0.0.1']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,false}, {record_name,schema}, {attributes,[table,cstruct]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1310,615376,847566},'ns_1@127.0.0.1'}}, {version,{{3,0},{'ns_1@127.0.0.1',{1310,615376,868870}}}}]}, {tid,3,<6093.86.0>}} INFO REPORT <6093.66.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.66.0>:mb_mnesia:194: Peers: ['ns_1@127.0.0.1'] INFO REPORT <6093.66.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.66.0>:mb_mnesia:194: Peers: ['ns_1@127.0.0.1'] INFO REPORT <6093.66.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.66.0>:mb_mnesia:199: Mnesia table event: {write,{schema,local_config, [{name,local_config}, {type,set}, {ram_copies,[]}, {disc_copies,['ns_1@127.0.0.1']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,local_config}, {attributes,[key,val]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1310,615379,105717},'ns_1@127.0.0.1'}}, {version,{{2,0},[]}}]}, {tid,5,<6093.111.0>}} INFO REPORT <6093.66.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.66.0>:mb_mnesia:199: Mnesia table event: {write,{schema,local_config, [{name,local_config}, {type,set}, {ram_copies,[]}, {disc_copies,['ns_1@127.0.0.1']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,local_config}, {attributes,[key,val]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1310,615379,105717},'ns_1@127.0.0.1'}}, {version,{{2,0},[]}}]}, {tid,5,<6093.111.0>}} PROGRESS REPORT <6093.64.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,mb_mnesia_sup} started [{pid,<6093.66.0>}, {name,mb_mnesia}, {mfargs,{mb_mnesia,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <6093.53.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<6093.64.0>}, {name,mb_mnesia_sup}, {mfargs,{mb_mnesia_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <6093.117.0> 2011-07-13 23:49:40 =============================================================================== loading static ns_config from "/opt/membase/etc/membase/config" PROGRESS REPORT <6093.117.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_config_sup} started [{pid,<6093.118.0>}, {name,ns_config_events}, {mfargs,{gen_event,start_link,[{local,ns_config_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <6093.119.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.119.0>:ns_config_default:215: Upgrading config from 1.6 to 1.7 INFO REPORT <6093.119.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.119.0>:ns_config:404: Upgrading config by changes: [{set,{node,'ns_1@127.0.0.1',config_version},{1,7}}, {set, {node,'ns_1@127.0.0.1',memcached}, [{port,11210}, {dbdir,"/opt/membase/var/lib/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"/opt/membase/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"/opt/membase/lib/memcached/ep.so"}, {initfile,"/opt/membase/etc/membase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine,"/opt/membase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}]}, {set, {node,'ns_1@127.0.0.1',ns_log}, [{filename,"/opt/membase/var/lib/membase/data/ns_log"}]}, {set,port_servers, [{moxi,"/opt/membase/bin/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{rest,port}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",{"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD",{"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,exit_status,port_server_send_eol,stderr_to_stdout, stream]}, {memcached,"/opt/membase/bin/memcached", ["-X","/opt/membase/lib/memcached/stdin_term_handler.so","-p", {"~B",[port]}, "-E","/opt/membase/lib/memcached/bucket_engine.so","-B","binary", "-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status,port_server_send_eol, stream]}]}, {set, {node,'ns_1@127.0.0.1',isasl}, [{path,"/opt/membase/var/lib/membase/data/isasl.pw"}]}, {set,directory,"/opt/membase/var/lib/membase/config"}] INFO REPORT <6093.119.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.119.0>:ns_config_default:239: Upgrading config from 1.7 to 1.7.1 INFO REPORT <6093.119.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.119.0>:ns_config:404: Upgrading config by changes: [{set,{node,'ns_1@127.0.0.1',config_version},{1,7,1}}, {set,email_alerts, [{recipients,["root@localhost"]}, {sender,"membase@localhost"}, {enabled,true}, {email_server,[{user,[]}, {pass,[]}, {host,"localhost"}, {port,25}, {encrypt,false}]}, {alerts,[auto_failover_node,auto_failover_maximum_reached, auto_failover_other_nodes_down, auto_failover_cluster_too_small]}]}, {set,auto_failover_cfg, [{enabled,false},{timeout,30},{max_nodes,1},{count,0}]}] INFO REPORT <6093.119.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.119.0>:ns_config:438: Upgraded initial config: {config, {full,"/opt/membase/etc/membase/config",undefined,ns_config_default}, [[], [{directory,"/opt/membase/var/lib/membase/config"}, {nodes_wanted,['ns_1@127.0.0.1']}, {{node,'ns_1@127.0.0.1',membership},active}, {{node,'ns_1@127.0.0.1',rest},[{port,8091}]}, {rest_creds,[{creds,[]}]}, {{node,'ns_1@127.0.0.1',isasl}, [{path,"/opt/membase/var/lib/membase/data/isasl.pw"}]}, {{node,'ns_1@127.0.0.1',memcached}, [{port,11210}, {dbdir,"/opt/membase/var/lib/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"/opt/membase/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"/opt/membase/lib/memcached/ep.so"}, {initfile,"/opt/membase/etc/membase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine,"/opt/membase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}]}, {memory_quota,492}, {buckets,[{configs,[]}]}, {{node,'ns_1@127.0.0.1',moxi},[{port,11211},{verbosity,[]}]}, {port_servers, [{moxi,"/opt/membase/bin/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{rest,port}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR", {"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD", {"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,exit_status,port_server_send_eol,stderr_to_stdout, stream]}, {memcached,"/opt/membase/bin/memcached", ["-X","/opt/membase/lib/memcached/stdin_term_handler.so","-p", {"~B",[port]}, "-E","/opt/membase/lib/memcached/bucket_engine.so","-B", "binary","-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status,port_server_send_eol, stream]}]}, {{node,'ns_1@127.0.0.1',ns_log}, [{filename,"/opt/membase/var/lib/membase/data/ns_log"}]}, {email_alerts, [{recipients,["root@localhost"]}, {sender,"membase@localhost"}, {enabled,true}, {email_server, [{user,[]}, {pass,[]}, {host,"localhost"}, {port,25}, {encrypt,false}]}, {alerts, [auto_failover_node,auto_failover_maximum_reached, auto_failover_other_nodes_down, auto_failover_cluster_too_small]}]}, {replication,[{enabled,true}]}, {auto_failover_cfg, [{enabled,false},{timeout,30},{max_nodes,1},{count,0}]}]], [[{directory,"/opt/membase/var/lib/membase/config"}, {{node,'ns_1@127.0.0.1',config_version},{1,7,1}}, {auto_failover_cfg, [{enabled,false},{timeout,30},{max_nodes,1},{count,0}]}, {buckets,[{configs,[]}]}, {email_alerts, [{recipients,["root@localhost"]}, {sender,"membase@localhost"}, {enabled,true}, {email_server, [{user,[]}, {pass,[]}, {host,"localhost"}, {port,25}, {encrypt,false}]}, {alerts, [auto_failover_node,auto_failover_maximum_reached, auto_failover_other_nodes_down, auto_failover_cluster_too_small]}]}, {memory_quota,492}, {nodes_wanted,['ns_1@127.0.0.1']}, {port_servers, [{moxi,"/opt/membase/bin/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{rest,port}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR", {"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD", {"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,exit_status,port_server_send_eol,stderr_to_stdout, stream]}, {memcached,"/opt/membase/bin/memcached", ["-X","/opt/membase/lib/memcached/stdin_term_handler.so","-p", {"~B",[port]}, "-E","/opt/membase/lib/memcached/bucket_engine.so","-B", "binary","-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status,port_server_send_eol, stream]}]}, {replication,[{enabled,true}]}, {rest_creds,[{creds,[]}]}, {{node,'ns_1@127.0.0.1',isasl}, [{path,"/opt/membase/var/lib/membase/data/isasl.pw"}]}, {{node,'ns_1@127.0.0.1',membership},active}, {{node,'ns_1@127.0.0.1',memcached}, [{port,11210}, {dbdir,"/opt/membase/var/lib/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"/opt/membase/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"/opt/membase/lib/memcached/ep.so"}, {initfile,"/opt/membase/etc/membase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine,"/opt/membase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}]}, {{node,'ns_1@127.0.0.1',moxi},[{port,11211},{verbosity,[]}]}, {{node,'ns_1@127.0.0.1',ns_log}, [{filename,"/opt/membase/var/lib/membase/data/ns_log"}]}, {{node,'ns_1@127.0.0.1',rest},[{port,8091}]}]], ns_config_default, {ns_config,save_config_sync,[]}, undefined,false} PROGRESS REPORT <6093.117.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_config_sup} started [{pid,<6093.119.0>}, {name,ns_config}, {mfargs,{ns_config,start_link, ["/opt/membase/etc/membase/config", ns_config_default]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6093.117.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_config_sup} started [{pid,<6093.122.0>}, {name,ns_config_remote}, {mfargs,{ns_config_replica,start_link,[{local,ns_config_remote}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <6093.123.0> 2011-07-13 23:49:40 =============================================================================== isasl_sync init: ["/opt/membase/var/lib/membase/data/isasl.pw","_admin", "_admin"] INFO REPORT <6093.123.0> 2011-07-13 23:49:40 =============================================================================== isasl_sync init buckets: [] INFO REPORT <6093.123.0> 2011-07-13 23:49:40 =============================================================================== Writing isasl passwd file: "/opt/membase/var/lib/membase/data/isasl.pw" PROGRESS REPORT <6093.117.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_config_sup} started [{pid,<6093.123.0>}, {name,ns_config_isasl_sync}, {mfargs,{ns_config_isasl_sync,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6093.117.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_config_sup} started [{pid,<6093.125.0>}, {name,ns_config_log}, {mfargs,{ns_config_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6093.53.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<6093.117.0>}, {name,ns_config_sup}, {mfargs,{ns_config_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <6093.127.0> 2011-07-13 23:49:40 =============================================================================== ns_log:init(): Couldn't load logs from "/opt/membase/var/lib/membase/data/ns_log": {error, enoent} PROGRESS REPORT <6093.126.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6093.127.0>}, {name,ns_log}, {mfargs,{ns_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6093.126.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6093.128.0>}, {name,ns_log_events}, {mfargs,{gen_event,start_link,[{local,ns_log_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6093.129.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<6093.130.0>}, {name,ns_node_disco_events}, {mfargs,{gen_event,start_link,[{local,ns_node_disco_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <6093.131.0> 2011-07-13 23:49:40 =============================================================================== Initting ns_node_disco with [] INFO REPORT <6093.132.0> 2011-07-13 23:49:40 =============================================================================== ns_node_disco cookie_sync INFO REPORT <6093.132.0> 2011-07-13 23:49:40 =============================================================================== ns_log: logging ns_node_disco:3:Initial otp cookie generated: bhxehmsjnydvwwbh INFO REPORT <6093.118.0> 2011-07-13 23:49:40 =============================================================================== config change: otp -> [{cookie,bhxehmsjnydvwwbh}] INFO REPORT <6093.132.0> 2011-07-13 23:49:40 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@127.0.0.1'], with cookie: bhxehmsjnydvwwbh INFO REPORT <6093.132.0> 2011-07-13 23:49:40 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@127.0.0.1'], with cookie: bhxehmsjnydvwwbh PROGRESS REPORT <6093.129.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<6093.131.0>}, {name,ns_node_disco}, {mfargs,{ns_node_disco,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6093.129.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<6093.135.0>}, {name,ns_node_disco_log}, {mfargs,{ns_node_disco_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6093.129.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<6093.136.0>}, {name,ns_node_disco_conf_events}, {mfargs,{ns_node_disco_conf_events,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <6093.137.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.137.0>:ns_config_rep:56: init pulling INFO REPORT <6093.137.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.137.0>:ns_config_rep:58: init pushing INFO REPORT <6093.137.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.137.0>:ns_config_rep:62: init reannouncing INFO REPORT <6093.118.0> 2011-07-13 23:49:40 =============================================================================== ns_node_disco_conf_events config on otp INFO REPORT <6093.118.0> 2011-07-13 23:49:40 =============================================================================== config change: otp -> [{cookie,bhxehmsjnydvwwbh}] INFO REPORT <6093.118.0> 2011-07-13 23:49:40 =============================================================================== config change: directory -> "/opt/membase/var/lib/membase/config" INFO REPORT <6093.118.0> 2011-07-13 23:49:40 =============================================================================== config change: {node,'ns_1@127.0.0.1',config_version} -> {1,7,1} INFO REPORT <6093.118.0> 2011-07-13 23:49:40 =============================================================================== config change: auto_failover_cfg -> [{enabled,false},{timeout,30},{max_nodes,1},{count,0}] INFO REPORT <6093.139.0> 2011-07-13 23:49:40 =============================================================================== ns_node_disco cookie_sync INFO REPORT <6093.118.0> 2011-07-13 23:49:40 =============================================================================== config change: buckets -> [{configs,[]}] INFO REPORT <6093.118.0> 2011-07-13 23:49:40 =============================================================================== config change: email_alerts -> [{recipients,["root@localhost"]}, {sender,"membase@localhost"}, {enabled,true}, {email_server,[{user,[]}, {pass,[]}, {host,"localhost"}, {port,25}, {encrypt,false}]}, {alerts,[auto_failover_node,auto_failover_maximum_reached, auto_failover_other_nodes_down,auto_failover_cluster_too_small]}] INFO REPORT <6093.118.0> 2011-07-13 23:49:40 =============================================================================== config change: memory_quota -> 492 INFO REPORT <6093.118.0> 2011-07-13 23:49:40 =============================================================================== ns_node_disco_conf_events config on nodes_wanted INFO REPORT <6093.118.0> 2011-07-13 23:49:40 =============================================================================== config change: nodes_wanted -> ['ns_1@127.0.0.1'] INFO REPORT <6093.118.0> 2011-07-13 23:49:40 =============================================================================== config change: port_servers -> [{moxi,"/opt/membase/bin/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{rest,port}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",{"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD",{"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,exit_status,port_server_send_eol,stderr_to_stdout,stream]}, {memcached,"/opt/membase/bin/memcached", ["-X","/opt/membase/lib/memcached/stdin_term_handler.so","-p", {"~B",[port]}, "-E","/opt/membase/lib/memcached/bucket_engine.so","-B","binary", "-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status,port_server_send_eol, stream]}] INFO REPORT <6093.139.0> 2011-07-13 23:49:40 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@127.0.0.1'], with cookie: bhxehmsjnydvwwbh INFO REPORT <6093.140.0> 2011-07-13 23:49:40 =============================================================================== ns_node_disco cookie_sync INFO REPORT <6093.118.0> 2011-07-13 23:49:40 =============================================================================== config change: replication -> [{enabled,true}] INFO REPORT <6093.118.0> 2011-07-13 23:49:40 =============================================================================== config change: rest_creds -> ******** INFO REPORT <6093.118.0> 2011-07-13 23:49:40 =============================================================================== config change: {node,'ns_1@127.0.0.1',isasl} -> [{path,"/opt/membase/var/lib/membase/data/isasl.pw"}] INFO REPORT <6093.118.0> 2011-07-13 23:49:40 =============================================================================== config change: {node,'ns_1@127.0.0.1',membership} -> active INFO REPORT <6093.139.0> 2011-07-13 23:49:40 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@127.0.0.1'], with cookie: bhxehmsjnydvwwbh INFO REPORT <6093.118.0> 2011-07-13 23:49:40 =============================================================================== config change: {node,'ns_1@127.0.0.1',memcached} -> [{port,11210}, {dbdir,"/opt/membase/var/lib/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"/opt/membase/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"/opt/membase/lib/memcached/ep.so"}, {initfile,"/opt/membase/etc/membase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine,"/opt/membase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}] INFO REPORT <6093.118.0> 2011-07-13 23:49:40 =============================================================================== config change: {node,'ns_1@127.0.0.1',moxi} -> [{port,11211},{verbosity,[]}] INFO REPORT <6093.140.0> 2011-07-13 23:49:40 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@127.0.0.1'], with cookie: bhxehmsjnydvwwbh INFO REPORT <6093.118.0> 2011-07-13 23:49:40 =============================================================================== config change: {node,'ns_1@127.0.0.1',ns_log} -> [{filename,"/opt/membase/var/lib/membase/data/ns_log"}] INFO REPORT <6093.118.0> 2011-07-13 23:49:40 =============================================================================== config change: {node,'ns_1@127.0.0.1',rest} -> [{port,8091}] INFO REPORT <6093.118.0> 2011-07-13 23:49:40 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <6093.140.0> 2011-07-13 23:49:40 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@127.0.0.1'], with cookie: bhxehmsjnydvwwbh INFO REPORT <6093.137.0> 2011-07-13 23:49:40 =============================================================================== Pushing config PROGRESS REPORT <6093.129.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<6093.137.0>}, {name,ns_config_rep}, {mfargs,{ns_config_rep,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6093.126.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6093.129.0>}, {name,ns_node_disco_sup}, {mfargs,{ns_node_disco_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <6093.126.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6093.142.0>}, {name,ns_tick_event}, {mfargs,{gen_event,start_link,[{local,ns_tick_event}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <6093.137.0> 2011-07-13 23:49:40 =============================================================================== Pushing config done INFO REPORT <6093.144.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.144.0>:mb_master:84: I'm the only node, so I'm the master. INFO REPORT <6093.145.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.145.0>:misc:784: start_singleton(gen_fsm, ns_orchestrator, [], []): started as <6093.146.0> on 'ns_1@127.0.0.1' PROGRESS REPORT <6093.145.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,mb_master_sup} started [{pid,<6093.146.0>}, {name,ns_orchestrator}, {mfargs,{ns_orchestrator,start_link,[]}}, {restart_type,permanent}, {shutdown,20}, {child_type,worker}] INFO REPORT <6093.145.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.145.0>:misc:784: start_singleton(gen_server, ns_tick, [], []): started as <6093.147.0> on 'ns_1@127.0.0.1' PROGRESS REPORT <6093.145.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,mb_master_sup} started [{pid,<6093.147.0>}, {name,ns_tick}, {mfargs,{ns_tick,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <6093.148.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.148.0>:auto_failover:120: init auto_failover. INFO REPORT <6093.145.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.145.0>:misc:784: start_singleton(gen_server, auto_failover, [], []): started as <6093.148.0> on 'ns_1@127.0.0.1' PROGRESS REPORT <6093.145.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,mb_master_sup} started [{pid,<6093.148.0>}, {name,auto_failover}, {mfargs,{auto_failover,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <6093.126.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6093.144.0>}, {name,mb_master}, {mfargs,{mb_master,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <6093.126.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6093.149.0>}, {name,buckets_events}, {mfargs,{gen_event,start_link,[{local,buckets_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6093.150.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_mail_sup} started [{pid,<6093.151.0>}, {name,ns_mail}, {mfargs,{ns_mail,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6093.150.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_mail_sup} started [{pid,<6093.152.0>}, {name,ns_mail_log}, {mfargs,{ns_mail_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6093.126.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6093.150.0>}, {name,ns_mail_sup}, {mfargs,{ns_mail_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <6093.128.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.128.0>:ns_mail_log:45: ns_mail_log started up PROGRESS REPORT <6093.126.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6093.153.0>}, {name,ns_heart}, {mfargs,{ns_heart,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6093.126.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6093.154.0>}, {name,ns_doctor}, {mfargs,{ns_doctor,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6093.156.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<6093.157.0>}, {name,menelaus_web}, {mfargs,{menelaus_web,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <6093.156.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<6093.159.0>}, {name,menelaus_event}, {mfargs,{menelaus_event,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <6093.156.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<6093.160.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <6093.156.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<6093.161.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <6093.126.0> 2011-07-13 23:49:40 =============================================================================== ns_log: logging menelaus_sup:1:Membase Server has started on web port 8091 on node 'ns_1@127.0.0.1'. PROGRESS REPORT <6093.126.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6093.156.0>}, {name,menelaus}, {mfargs,{menelaus_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <6093.162.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<6093.163.0>}, {name,ns_port_init}, {mfargs,{ns_port_init,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <6093.164.0> 2011-07-13 23:49:40 =============================================================================== starting ns_port_server with delay of 5000 PROGRESS REPORT <6093.162.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<6093.164.0>}, {name, {moxi,"/opt/membase/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",[]}, {"MOXI_SASL_PLAIN_PWD",[]}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]}}, {mfargs, {supervisor_cushion,start_link, [moxi,5000,ns_port_server,start_link, [moxi,"/opt/membase/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",[]}, {"MOXI_SASL_PLAIN_PWD",[]}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]]]}}, {restart_type,permanent}, {shutdown,10000}, {child_type,worker}] INFO REPORT <6093.166.0> 2011-07-13 23:49:40 =============================================================================== starting ns_port_server with delay of 5000 PROGRESS REPORT <6093.162.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<6093.166.0>}, {name, {memcached,"/opt/membase/bin/memcached", ["-X","/opt/membase/lib/memcached/stdin_term_handler.so", "-p","11210","-E", "/opt/membase/lib/memcached/bucket_engine.so","-B", "binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE", "/opt/membase/var/lib/membase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status, port_server_send_eol,stream]}}, {mfargs, {supervisor_cushion,start_link, [memcached,5000,ns_port_server,start_link, [memcached,"/opt/membase/bin/memcached", ["-X","/opt/membase/lib/memcached/stdin_term_handler.so", "-p","11210","-E", "/opt/membase/lib/memcached/bucket_engine.so","-B", "binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE", "/opt/membase/var/lib/membase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status, port_server_send_eol,stream]]]}}, {restart_type,permanent}, {shutdown,10000}, {child_type,worker}] PROGRESS REPORT <6093.126.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6093.162.0>}, {name,ns_port_sup}, {mfargs,{ns_port_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,60000}, {child_type,worker}] PROGRESS REPORT <6093.126.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6093.168.0>}, {name,ns_stats_event}, {mfargs,{gen_event,start_link,[{local,ns_stats_event}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6093.126.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6093.170.0>}, {name,ns_bucket_worker}, {mfargs,{work_queue,start_link,[ns_bucket_worker]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6093.126.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6093.171.0>}, {name,ns_bucket_sup}, {mfargs,{ns_bucket_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] ERROR REPORT <6093.153.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.153.0>:ns_heart:137: Failed to grab system stats: {'EXIT',{noproc,{gen_server,call, [{'stats_reader-@system','ns_1@127.0.0.1'}, {latest,"minute"}]}}} PROGRESS REPORT <6093.126.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6093.172.0>}, {name,system_stats_collector}, {mfargs,{system_stats_collector,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6093.126.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6093.173.0>}, {name,{stats_archiver,"@system"}}, {mfargs,{stats_archiver,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <6093.66.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.66.0>:mb_mnesia:109: Creating table 'stats_archiver-@system-minute' PROGRESS REPORT <6093.126.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6093.174.0>}, {name,{stats_reader,"@system"}}, {mfargs,{stats_reader,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6093.126.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6093.177.0>}, {name,ns_moxi_sup_work_queue}, {mfargs,{work_queue,start_link,[ns_moxi_sup_work_queue]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <6093.126.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<6093.178.0>}, {name,ns_moxi_sup}, {mfargs,{ns_moxi_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <6093.53.0> 2011-07-13 23:49:40 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<6093.126.0>}, {name,ns_server_sup}, {mfargs,{ns_server_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <6093.7.0> 2011-07-13 23:49:40 =============================================================================== application ns_server started_at 'ns_1@127.0.0.1' INFO REPORT <6093.154.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.154.0>:ns_doctor:82: Got initial status [{'ns_1@127.0.0.1', [{last_heard, {1310,615380, 373503}}, {active_buckets, []}, {ready_buckets, []}, {replication, []}, {memory, [{total, 21096712}, {processes, 3562808}, {processes_used, 3550408}, {system, 17533904}, {atom, 768425}, {atom_used, 743125}, {binary, 55920}, {code, 6978090}, {ets, 497880}]}, {system_stats, [{cpu_utilization_rate, 0}, {swap_total, 0}, {swap_used, 0}]}, {interesting_stats, []}, {cluster_compatibility_version, 1}, {version, [{kernel, "2.14.3"}, {mnesia, "4.4.17"}, {stdlib, "1.17.3"}, {os_mon, "2.2.5"}, {ns_server, "1.7.1r-42-gd713cca"}, {sasl, "2.1.9.3"}]}, {system_arch, "x86_64-unknown-linux-gnu"}, {wall_clock,3}, {memory_data, {644874240, 618971136, {<6093.7.0>, 196872}}}, {disk_data, [{"/", 5904748,66}, {"/dev/shm", 314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 20948 kB\nBuffers: 19848 kB\nCached: 427660 kB\nSwapCached: 0 kB\nActive: 207848 kB\nInactive: 325764 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 20948 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 62384 kB\nWriteback: 0 kB\nAnonPages: 86116 kB\nMapped: 10948 kB\nSlab: 36136 kB\nPageTables: 3440 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 384952 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory, 644874240}, {free_swap, 0}, {total_swap, 0}, {cached_memory, 437923840}, {buffered_memory, 20324352}, {free_memory, 21585920}, {total_memory, 644874240}]}, {statistics, [{wall_clock, {3890, 3598}}, {context_switches, {8419,0}}, {garbage_collection, {1378, 2725026, 0}}, {io, {{input, 6483779}, {output, 1469051}}}, {reductions, {1331800, 1331800}}, {run_queue, 0}, {runtime, {250, 250}}]}]}] INFO REPORT <6093.66.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.66.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-minute', [{name,'stats_archiver-@system-minute'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@127.0.0.1']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1310,615380,371521},'ns_1@127.0.0.1'}}, {version,{{2,0},[]}}]}, {tid,6,<6093.176.0>}} INFO REPORT <6093.66.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.66.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-minute', [{name,'stats_archiver-@system-minute'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@127.0.0.1']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1310,615380,371521},'ns_1@127.0.0.1'}}, {version,{{2,0},[]}}]}, {tid,6,<6093.176.0>}} INFO REPORT <6093.66.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.66.0>:mb_mnesia:109: Creating table 'stats_archiver-@system-hour' INFO REPORT <6093.66.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.66.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-hour', [{name,'stats_archiver-@system-hour'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@127.0.0.1']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1310,615380,509276},'ns_1@127.0.0.1'}}, {version,{{2,0},[]}}]}, {tid,7,<6093.185.0>}} INFO REPORT <6093.66.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.66.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-hour', [{name,'stats_archiver-@system-hour'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@127.0.0.1']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1310,615380,509276},'ns_1@127.0.0.1'}}, {version,{{2,0},[]}}]}, {tid,7,<6093.185.0>}} INFO REPORT <6093.66.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.66.0>:mb_mnesia:109: Creating table 'stats_archiver-@system-day' INFO REPORT <6093.66.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.66.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-day', [{name,'stats_archiver-@system-day'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@127.0.0.1']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1310,615380,670084},'ns_1@127.0.0.1'}}, {version,{{2,0},[]}}]}, {tid,8,<6093.193.0>}} INFO REPORT <6093.66.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.66.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-day', [{name,'stats_archiver-@system-day'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@127.0.0.1']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1310,615380,670084},'ns_1@127.0.0.1'}}, {version,{{2,0},[]}}]}, {tid,8,<6093.193.0>}} INFO REPORT <6093.66.0> 2011-07-13 23:49:40 =============================================================================== ns_1@127.0.0.1:<6093.66.0>:mb_mnesia:109: Creating table 'stats_archiver-@system-week' INFO REPORT <6093.66.0> 2011-07-13 23:49:41 =============================================================================== ns_1@127.0.0.1:<6093.66.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-week', [{name,'stats_archiver-@system-week'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@127.0.0.1']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1310,615380,848246},'ns_1@127.0.0.1'}}, {version,{{2,0},[]}}]}, {tid,9,<6093.203.0>}} INFO REPORT <6093.66.0> 2011-07-13 23:49:41 =============================================================================== ns_1@127.0.0.1:<6093.66.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-week', [{name,'stats_archiver-@system-week'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@127.0.0.1']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1310,615380,848246},'ns_1@127.0.0.1'}}, {version,{{2,0},[]}}]}, {tid,9,<6093.203.0>}} INFO REPORT <6093.66.0> 2011-07-13 23:49:41 =============================================================================== ns_1@127.0.0.1:<6093.66.0>:mb_mnesia:109: Creating table 'stats_archiver-@system-month' INFO REPORT <6093.66.0> 2011-07-13 23:49:41 =============================================================================== ns_1@127.0.0.1:<6093.66.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-month', [{name,'stats_archiver-@system-month'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@127.0.0.1']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1310,615381,125624},'ns_1@127.0.0.1'}}, {version,{{2,0},[]}}]}, {tid,10,<6093.210.0>}} INFO REPORT <6093.66.0> 2011-07-13 23:49:41 =============================================================================== ns_1@127.0.0.1:<6093.66.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-month', [{name,'stats_archiver-@system-month'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@127.0.0.1']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1310,615381,125624},'ns_1@127.0.0.1'}}, {version,{{2,0},[]}}]}, {tid,10,<6093.210.0>}} INFO REPORT <6093.66.0> 2011-07-13 23:49:41 =============================================================================== ns_1@127.0.0.1:<6093.66.0>:mb_mnesia:109: Creating table 'stats_archiver-@system-year' INFO REPORT <6093.66.0> 2011-07-13 23:49:41 =============================================================================== ns_1@127.0.0.1:<6093.66.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-year', [{name,'stats_archiver-@system-year'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@127.0.0.1']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1310,615381,166060},'ns_1@127.0.0.1'}}, {version,{{2,0},[]}}]}, {tid,11,<6093.217.0>}} INFO REPORT <6093.66.0> 2011-07-13 23:49:41 =============================================================================== ns_1@127.0.0.1:<6093.66.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-year', [{name,'stats_archiver-@system-year'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@127.0.0.1']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1310,615381,166060},'ns_1@127.0.0.1'}}, {version,{{2,0},[]}}]}, {tid,11,<6093.217.0>}} PROGRESS REPORT <6093.226.0> 2011-07-13 23:49:43 =============================================================================== supervisor {local,inet_gethost_native_sup} started [{pid,<6093.227.0>},{mfa,{inet_gethost_native,init,[[]]}}] PROGRESS REPORT <6093.26.0> 2011-07-13 23:49:43 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<6093.226.0>}, {name,inet_gethost_native_sup}, {mfargs,{inet_gethost_native,start_link,[]}}, {restart_type,temporary}, {shutdown,1000}, {child_type,worker}] INFO REPORT <6093.118.0> 2011-07-13 23:49:45 =============================================================================== unsupervising port: {moxi,"/opt/membase/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env,[{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",[]}, {"MOXI_SASL_PLAIN_PWD",[]}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]} INFO REPORT <6093.158.0> 2011-07-13 23:49:45 =============================================================================== menelaus_web streaming socket closed by client INFO REPORT <6093.165.0> 2011-07-13 23:49:45 =============================================================================== moxi<0.165.0>: EOL on stdin. Exiting INFO REPORT <6093.118.0> 2011-07-13 23:49:45 =============================================================================== supervising port: {moxi,"/opt/membase/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env,[{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]} INFO REPORT <6093.236.0> 2011-07-13 23:49:45 =============================================================================== starting ns_port_server with delay of 5000 PROGRESS REPORT <6093.162.0> 2011-07-13 23:49:45 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<6093.236.0>}, {name, {moxi,"/opt/membase/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]}}, {mfargs, {supervisor_cushion,start_link, [moxi,5000,ns_port_server,start_link, [moxi,"/opt/membase/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]]]}}, {restart_type,permanent}, {shutdown,10000}, {child_type,worker}] INFO REPORT <6093.118.0> 2011-07-13 23:49:45 =============================================================================== config change: rest -> [{port,8091}] INFO REPORT <6093.118.0> 2011-07-13 23:49:45 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <6093.137.0> 2011-07-13 23:49:45 =============================================================================== Pushing config INFO REPORT <6093.137.0> 2011-07-13 23:49:45 =============================================================================== Pushing config done INFO REPORT <6093.118.0> 2011-07-13 23:49:45 =============================================================================== config change: rest_creds -> ******** INFO REPORT <6093.118.0> 2011-07-13 23:49:45 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <6093.137.0> 2011-07-13 23:49:45 =============================================================================== Pushing config INFO REPORT <6093.137.0> 2011-07-13 23:49:45 =============================================================================== Pushing config done INFO REPORT <6093.237.0> 2011-07-13 23:49:46 =============================================================================== moxi<0.237.0>: 2011-07-13 23:49:45: (cproxy_config.c.317) env: MOXI_SASL_PLAIN_USR (13) moxi<0.237.0>: 2011-07-13 23:49:45: (cproxy_config.c.326) env: MOXI_SASL_PLAIN_PWD (8) INFO REPORT <6093.154.0> 2011-07-13 23:50:40 =============================================================================== ns_1@127.0.0.1:<6093.154.0>:ns_doctor:86: Current node statuses: [{'ns_1@127.0.0.1', [{last_heard,{1310,615435,310779}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[]}, {memory, [{total,22291664}, {processes,4188656}, {processes_used,4174240}, {system,18103008}, {atom,784585}, {atom_used,764895}, {binary,94976}, {code,7369034}, {ets,607048}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,58}, {memory_data,{644874240,618971136,{<6093.7.0>,196872}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 10948 kB\nBuffers: 19980 kB\nCached: 427764 kB\nSwapCached: 0 kB\nActive: 217224 kB\nInactive: 324920 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 10948 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 36160 kB\nWriteback: 0 kB\nAnonPages: 94400 kB\nMapped: 11496 kB\nSlab: 36304 kB\nPageTables: 3896 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 538212 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,438030336}, {buffered_memory,20459520}, {free_memory,11210752}, {total_memory,644874240}]}, {statistics, [{wall_clock,{48850,1}}, {context_switches,{14014,0}}, {garbage_collection,{2589,4521349,0}}, {io,{{input,6822760},{output,1580533}}}, {reductions,{1853731,152058}}, {run_queue,0}, {runtime,{270,0}}]}]}] INFO REPORT <6093.154.0> 2011-07-13 23:51:40 =============================================================================== ns_1@127.0.0.1:<6093.154.0>:ns_doctor:86: Current node statuses: [{'ns_1@127.0.0.1', [{last_heard,{1310,615495,310531}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[]}, {memory, [{total,22157480}, {processes,4039072}, {processes_used,4024656}, {system,18118408}, {atom,784585}, {atom_used,764895}, {binary,78744}, {code,7369034}, {ets,637336}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,118}, {memory_data,{644874240,633663488,{<6093.162.0>,371936}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 10956 kB\nBuffers: 20052 kB\nCached: 427840 kB\nSwapCached: 0 kB\nActive: 217332 kB\nInactive: 325004 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 10956 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 17408 kB\nWriteback: 0 kB\nAnonPages: 94444 kB\nMapped: 11496 kB\nSlab: 36224 kB\nPageTables: 3896 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 538212 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,438108160}, {buffered_memory,20533248}, {free_memory,11218944}, {total_memory,644874240}]}, {statistics, [{wall_clock,{108850,1}}, {context_switches,{19704,0}}, {garbage_collection,{3717,6255286,0}}, {io,{{input,6868371},{output,1666923}}}, {reductions,{2442530,169073}}, {run_queue,0}, {runtime,{270,0}}]}]}] INFO REPORT <6093.154.0> 2011-07-13 23:52:40 =============================================================================== ns_1@127.0.0.1:<6093.154.0>:ns_doctor:86: Current node statuses: [{'ns_1@127.0.0.1', [{last_heard,{1310,615555,310280}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[]}, {memory, [{total,22222096}, {processes,4061616}, {processes_used,4047200}, {system,18160480}, {atom,784585}, {atom_used,764895}, {binary,91872}, {code,7369034}, {ets,664936}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,178}, {memory_data,{644874240,633655296,{<6093.162.0>,371936}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 10956 kB\nBuffers: 20132 kB\nCached: 427924 kB\nSwapCached: 0 kB\nActive: 217488 kB\nInactive: 325124 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 10956 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 9140 kB\nWriteback: 0 kB\nAnonPages: 94560 kB\nMapped: 11500 kB\nSlab: 36240 kB\nPageTables: 3896 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 538212 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,438194176}, {buffered_memory,20615168}, {free_memory,11218944}, {total_memory,644874240}]}, {statistics, [{wall_clock,{168849,0}}, {context_switches,{25461,0}}, {garbage_collection,{4842,7999613,0}}, {io,{{input,6913991},{output,1758894}}}, {reductions,{3040866,159101}}, {run_queue,0}, {runtime,{270,0}}]}]}] INFO REPORT <6093.63.0> 2011-07-13 23:53:28 =============================================================================== rb: ERROR! A report on bad form was encountered. It can not be printed to the log. Details: "2011-07-13 23:53:28" {{{2011,7,13},{23,53,28}}, {info_msg,<6093.50.0>, {<6093.63.0>, "~s:~p:~s:~B: handling engage_cluster(..)~n~n", ['ns_1@127.0.0.1',<6093.63.0>,ns_cluster,96, [{<<"availableStorage">>, {struct, [{<<"hdd">>, [{struct, [{<<"path">>,<<"/">>}, {<<"sizeKBytes">>,5904748}, {<<"usagePercent">>,66}]}, {struct, [{<<"path">>,<<"/dev/shm">>}, {<<"sizeKBytes">>,314880}, {<<"usagePercent">>,0}]}]}]}}, {<<"memoryQuota">>,492}, {<<"storageTotals">>, {struct, [{<<"ram">>, {struct, [{<<"usedByData">>,0}, {<<"total">>,644874240}, {<<"quotaTotal">>,515899392}, {<<"used">>,633511936}]}}, {<<"hdd">>, {struct, [{<<"usedByData">>,0}, {<<"total">>,6046461952.0}, {<<"quotaTotal">>,6046461952.0}, {<<"used">>,3990664888.0}, {<<"free">>,2055797064}]}}]}}, {<<"storage">>, {struct, [{<<"ssd">>,[]}, {<<"hdd">>, [{struct, [{<<"path">>, <<"/opt/membase/var/lib/membase/data">>}, {<<"quotaMb">>,<<"none">>}, {<<"state">>,<<"ok">>}]}]}]}}, {<<"systemStats">>, {struct, [{<<"cpu_utilization_rate">>,0}, {<<"swap_total">>,0}, {<<"swap_used">>,0}]}}, {<<"interestingStats">>,{struct,[]}}, {<<"uptime">>,<<"331">>}, {<<"memoryTotal">>,644874240}, {<<"memoryFree">>,11362304}, {<<"mcdMemoryReserved">>,492}, {<<"mcdMemoryAllocated">>,492}, {<<"otpNode">>,<<"ns_1@10.88.23.62">>}, {<<"otpCookie">>,<<"brohpzslpcmrmtnv">>}, {<<"clusterMembership">>,<<"active">>}, {<<"status">>,<<"healthy">>}, {<<"hostname">>,<<"10.88.23.62:8091">>}, {<<"clusterCompatibility">>,1}, {<<"version">>,<<"1.7.1r-42-gd713cca">>}, {<<"os">>,<<"x86_64-unknown-linux-gnu">>}, {<<"ports">>, {struct, [{<<"proxy">>,11211},{<<"direct">>,11210}]}}]]}}} rb: Logging resumed... INFO REPORT <6093.63.0> 2011-07-13 23:53:29 =============================================================================== ns_1@127.0.0.1:<6093.63.0>:ns_cluster:247: Decided to change address to "10.88.165.188" INFO REPORT <0.131.0> 2011-07-13 23:53:29 =============================================================================== ns_log: logging ns_node_disco:5:Node nonode@nohost saw that node 'ns_1@127.0.0.1' went down. INFO REPORT <0.130.0> 2011-07-13 23:53:29 =============================================================================== ns_node_disco_log: nodes changed: [] INFO REPORT <0.56.0> 2011-07-13 23:53:29 =============================================================================== Adjusted IP to "10.88.165.188" INFO REPORT <0.56.0> 2011-07-13 23:53:29 =============================================================================== nonode@nohost:<0.56.0>:dist_manager:105: Attempting to bring up net_kernel with name 'ns_1@10.88.165.188' PROGRESS REPORT <0.759.0> 2011-07-13 23:53:29 =============================================================================== supervisor {local,net_sup} started [{pid,<0.760.0>}, {name,erl_epmd}, {mfargs,{erl_epmd,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] PROGRESS REPORT <0.759.0> 2011-07-13 23:53:29 =============================================================================== supervisor {local,net_sup} started [{pid,<0.761.0>}, {name,auth}, {mfargs,{auth,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] INFO REPORT <5901.131.0> 2011-07-13 23:53:29 =============================================================================== ns_log: logging ns_node_disco:4:Node 'ns_1@10.88.165.188' saw that node 'ns_1@10.88.165.188' came up. PROGRESS REPORT <5901.759.0> 2011-07-13 23:53:29 =============================================================================== supervisor {local,net_sup} started [{pid,<5901.762.0>}, {name,net_kernel}, {mfargs,{net_kernel,start_link,[['ns_1@10.88.165.188',longnames]]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] PROGRESS REPORT <5901.11.0> 2011-07-13 23:53:29 =============================================================================== supervisor {local,kernel_sup} started [{pid,<5901.759.0>}, {name,net_sup_dynamic}, {mfargs,{erl_distribution,start_link, [['ns_1@10.88.165.188',longnames]]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,supervisor}] INFO REPORT <5901.56.0> 2011-07-13 23:53:29 =============================================================================== Re-setting cookie {bhxehmsjnydvwwbh,'ns_1@10.88.165.188'} INFO REPORT <5901.56.0> 2011-07-13 23:53:29 =============================================================================== saving ip config to "/opt/membase/var/lib/membase/ip" INFO REPORT <5901.56.0> 2011-07-13 23:53:29 =============================================================================== save_address_config: ok INFO REPORT <5901.66.0> 2011-07-13 23:53:29 =============================================================================== ns_1@10.88.165.188:<5901.66.0>:mb_mnesia:132: Renaming node from 'ns_1@127.0.0.1' to 'ns_1@10.88.165.188'. INFO REPORT <5901.7.0> 2011-07-13 23:53:29 =============================================================================== application mnesia exited stopped type temporary INFO REPORT <5901.66.0> 2011-07-13 23:53:29 =============================================================================== ns_1@10.88.165.188:<5901.66.0>:mb_mnesia:232: Found backup. Restoring Mnesia database. PROGRESS REPORT <5901.782.0> 2011-07-13 23:53:29 =============================================================================== supervisor {local,mnesia_sup} started [{pid,<5901.783.0>}, {name,mnesia_event}, {mfargs,{mnesia_sup,start_event,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] PROGRESS REPORT <5901.784.0> 2011-07-13 23:53:29 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.785.0>}, {name,mnesia_monitor}, {mfargs,{mnesia_monitor,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <5901.784.0> 2011-07-13 23:53:29 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.786.0>}, {name,mnesia_subscr}, {mfargs,{mnesia_subscr,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <5901.784.0> 2011-07-13 23:53:29 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.787.0>}, {name,mnesia_locker}, {mfargs,{mnesia_locker,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <5901.784.0> 2011-07-13 23:53:29 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.788.0>}, {name,mnesia_recover}, {mfargs,{mnesia_recover,start,[]}}, {restart_type,permanent}, {shutdown,180000}, {child_type,worker}] PROGRESS REPORT <5901.784.0> 2011-07-13 23:53:29 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.789.0>}, {name,mnesia_tm}, {mfargs,{mnesia_tm,start,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] ERROR REPORT <5901.173.0> 2011-07-13 23:53:29 =============================================================================== ** Generic server 'stats_archiver-@system' terminating ** Last message in was {stats,"@system", {stat_entry,1310615609302, [{cpu_idle_ms,980}, {cpu_local_ms,1000}, {cpu_utilization_rate,2.0}, {mem_actual_free,469291008}, {mem_actual_used,175583232}, {mem_free,10260480}, {mem_total,644874240}, {mem_used,634613760}, {swap_total,0}, {swap_used,0}]}} ** When Server state == {state,"@system"} ** Reason for termination == ** {{badmatch,{aborted,{no_exists,'stats_archiver-@system-minute'}}}, [{stats_archiver,do_handle_info,2}, {stats_archiver,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.173.0> 2011-07-13 23:53:29 =============================================================================== Crashing process initial_call {stats_archiver,init,['Argument__1']} pid <5901.173.0> registered_name 'stats_archiver-@system' error_info {exit, {{badmatch,{aborted,{no_exists,'stats_archiver-@system-minute'}}}, [{stats_archiver,do_handle_info,2}, {stats_archiver,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors [ns_server_sup,ns_server_cluster_sup,<5901.51.0>] messages [] links [<5901.126.0>,<5901.168.0>,<5901.79.0>] dictionary [] trap_exit false status running heap_size 4181 stack_size 24 reductions 1098742 SUPERVISOR REPORT <5901.126.0> 2011-07-13 23:53:29 =============================================================================== Reporting supervisor {local,ns_server_sup} Child process errorContext child_terminated reason {{badmatch,{aborted,{no_exists,'stats_archiver-@system-minute'}}}, [{stats_archiver,do_handle_info,2}, {stats_archiver,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.173.0> name {stats_archiver,"@system"} mfargs {stats_archiver,start_link,["@system"]} restart_type permanent shutdown 1000 child_type worker PROGRESS REPORT <5901.784.0> 2011-07-13 23:53:29 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.832.0>}, {name,mnesia_checkpoint_sup}, {mfargs,{mnesia_checkpoint_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.784.0> 2011-07-13 23:53:29 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.834.0>}, {name,mnesia_snmp_sup}, {mfargs,{mnesia_snmp_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.784.0> 2011-07-13 23:53:29 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.835.0>}, {name,mnesia_controller}, {mfargs,{mnesia_controller,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <5901.784.0> 2011-07-13 23:53:29 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.836.0>}, {name,mnesia_late_loader}, {mfargs,{mnesia_late_loader,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <5901.782.0> 2011-07-13 23:53:29 =============================================================================== supervisor {local,mnesia_sup} started [{pid,<5901.784.0>}, {name,mnesia_kernel_sup}, {mfargs,{mnesia_kernel_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.126.0> 2011-07-13 23:53:29 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.833.0>}, {name,{stats_archiver,"@system"}}, {mfargs,{stats_archiver,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.7.0> 2011-07-13 23:53:29 =============================================================================== application mnesia started_at 'ns_1@10.88.165.188' INFO REPORT <5901.66.0> 2011-07-13 23:53:29 =============================================================================== ns_1@10.88.165.188:<5901.66.0>:mb_mnesia:245: Current config: [{access_module,mnesia}, {auto_repair,true}, {backup_module,mnesia_backup}, {checkpoints,[]}, {db_nodes,['ns_1@10.88.165.188']}, {debug,verbose}, {directory,"/opt/membase/var/lib/membase/mnesia"}, {dump_log_load_regulation,false}, {dump_log_time_threshold,180000}, {dump_log_update_in_place,true}, {dump_log_write_threshold,1000}, {embedded_mnemosyne,false}, {event_module,mnesia_event}, {extra_db_nodes,[]}, {fallback_activated,false}, {held_locks,[]}, {ignore_fallback_at_startup,false}, {fallback_error_function,{mnesia,lkill}}, {is_running,yes}, {local_tables,[local_config,'stats_archiver-@system-day',cluster, 'stats_archiver-@system-week','stats_archiver-@system-minute', 'stats_archiver-@system-year',schema, 'stats_archiver-@system-month','stats_archiver-@system-hour']}, {lock_queue,[]}, {log_version,"4.3"}, {master_node_tables,[]}, {max_wait_for_decision,10000}, {protocol_version,{7,6}}, {running_db_nodes,['ns_1@10.88.165.188']}, {schema_location,opt_disc}, {schema_version,{3,0}}, {subscribers,[<5901.783.0>,<5901.66.0>]}, {tables,[local_config,'stats_archiver-@system-day',cluster, 'stats_archiver-@system-week','stats_archiver-@system-minute', 'stats_archiver-@system-year',schema,'stats_archiver-@system-month', 'stats_archiver-@system-hour']}, {transaction_commits,2}, {transaction_failures,1}, {transaction_log_writes,0}, {transaction_restarts,0}, {transactions,[]}, {use_dir,true}, {core_dir,false}, {no_table_loaders,2}, {dc_dump_limit,4}, {send_compressed,0}, {version,"4.4.17"}] Peers: ['ns_1@10.88.165.188'] INFO REPORT <5901.66.0> 2011-07-13 23:53:29 =============================================================================== ns_1@10.88.165.188:<5901.66.0>:mb_mnesia:166: Info from Mnesia: {mnesia_checkpoint,{{1310,615609,200489},'ns_1@127.0.0.1'}} starting: <5901.753.0> INFO REPORT <5901.66.0> 2011-07-13 23:53:29 =============================================================================== ns_1@10.88.165.188:<5901.66.0>:mb_mnesia:166: Info from Mnesia: {mnesia_checkpoint,{{1310,615609,200489},'ns_1@127.0.0.1'}} terminated: shutdown INFO REPORT <5901.66.0> 2011-07-13 23:53:29 =============================================================================== ns_1@10.88.165.188:<5901.66.0>:mb_mnesia:166: Info from Mnesia: mnesia_controller terminated: shutdown INFO REPORT <5901.66.0> 2011-07-13 23:53:29 =============================================================================== ns_1@10.88.165.188:<5901.66.0>:mb_mnesia:166: Info from Mnesia: mnesia_tm terminated: shutdown INFO REPORT <5901.66.0> 2011-07-13 23:53:29 =============================================================================== ns_1@10.88.165.188:<5901.66.0>:mb_mnesia:166: Info from Mnesia: mnesia_recover terminated: shutdown INFO REPORT <5901.66.0> 2011-07-13 23:53:29 =============================================================================== ns_1@10.88.165.188:<5901.66.0>:mb_mnesia:166: Info from Mnesia: mnesia_locker terminated: shutdown INFO REPORT <5901.66.0> 2011-07-13 23:53:29 =============================================================================== ns_1@10.88.165.188:<5901.66.0>:mb_mnesia:169: Saw Mnesia go down on 'ns_1@10.88.165.188' INFO REPORT <5901.119.0> 2011-07-13 23:53:29 =============================================================================== renaming node conf {node,'ns_1@127.0.0.1',config_version} -> {node, 'ns_1@10.88.165.188', config_version}: {1,7,1} -> {1,7,1} INFO REPORT <5901.119.0> 2011-07-13 23:53:29 =============================================================================== renaming node conf nodes_wanted -> nodes_wanted: ['ns_1@127.0.0.1'] -> ['ns_1@10.88.165.188'] INFO REPORT <5901.119.0> 2011-07-13 23:53:29 =============================================================================== renaming node conf {node,'ns_1@127.0.0.1',isasl} -> {node, 'ns_1@10.88.165.188', isasl}: [{path,"/opt/membase/var/lib/membase/data/isasl.pw"}] -> [{path,"/opt/membase/var/lib/membase/data/isasl.pw"}] INFO REPORT <5901.119.0> 2011-07-13 23:53:29 =============================================================================== renaming node conf {node,'ns_1@127.0.0.1',membership} -> {node, 'ns_1@10.88.165.188', membership}: active -> active INFO REPORT <5901.119.0> 2011-07-13 23:53:29 =============================================================================== renaming node conf {node,'ns_1@127.0.0.1',memcached} -> {node, 'ns_1@10.88.165.188', memcached}: [{port,11210}, {dbdir,"/opt/membase/var/lib/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"/opt/membase/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"/opt/membase/lib/memcached/ep.so"}, {initfile,"/opt/membase/etc/membase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine,"/opt/membase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}] -> [{port,11210}, {dbdir,"/opt/membase/var/lib/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"/opt/membase/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"/opt/membase/lib/memcached/ep.so"}, {initfile,"/opt/membase/etc/membase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine,"/opt/membase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}] INFO REPORT <5901.119.0> 2011-07-13 23:53:29 =============================================================================== renaming node conf {node,'ns_1@127.0.0.1',moxi} -> {node, 'ns_1@10.88.165.188',moxi}: [{port,11211},{verbosity,[]}] -> [{port,11211},{verbosity,[]}] INFO REPORT <5901.119.0> 2011-07-13 23:53:29 =============================================================================== renaming node conf {node,'ns_1@127.0.0.1',ns_log} -> {node, 'ns_1@10.88.165.188', ns_log}: [{filename,"/opt/membase/var/lib/membase/data/ns_log"}] -> [{filename,"/opt/membase/var/lib/membase/data/ns_log"}] INFO REPORT <5901.119.0> 2011-07-13 23:53:29 =============================================================================== renaming node conf {node,'ns_1@127.0.0.1',rest} -> {node, 'ns_1@10.88.165.188',rest}: [{port,8091}] -> [{port,8091}] PROGRESS REPORT <5901.126.0> 2011-07-13 23:53:29 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.846.0>}, {name,ns_doctor}, {mfargs,{ns_doctor,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.63.0> 2011-07-13 23:53:29 =============================================================================== ns_1@10.88.165.188:<5901.63.0>:ns_cluster:254: Renamed node. New name is 'ns_1@10.88.165.188'. INFO REPORT <5901.118.0> 2011-07-13 23:53:29 =============================================================================== config change: {node,'ns_1@10.88.165.188',rest} -> [{port,8091}] ERROR REPORT <5901.153.0> 2011-07-13 23:53:29 =============================================================================== ns_1@10.88.165.188:<5901.153.0>:ns_heart:137: Failed to grab system stats: {error,{exit,{aborted,{no_exists,['stats_archiver-@system-minute']}}}} INFO REPORT <5901.63.0> 2011-07-13 23:53:29 =============================================================================== ns_1@10.88.165.188:<5901.63.0>:ns_cluster:98: engage_cluster(..) -> {ok,ok} INFO REPORT <5901.118.0> 2011-07-13 23:53:29 =============================================================================== config change: {node,'ns_1@10.88.165.188',ns_log} -> [{filename,"/opt/membase/var/lib/membase/data/ns_log"}] INFO REPORT <5901.846.0> 2011-07-13 23:53:29 =============================================================================== ns_1@10.88.165.188:<5901.846.0>:ns_doctor:82: Got initial status [{'ns_1@10.88.165.188', [{last_heard, {1310, 615609, 510290}}, {active_buckets, []}, {ready_buckets, []}, {replication, []}, {memory, [{total, 23285752}, {processes, 5125008}, {processes_used, 5116504}, {system, 18160744}, {atom, 784585}, {atom_used, 765746}, {binary, 188600}, {code, 7382775}, {ets, 541856}]}, {system_stats, [{cpu_utilization_rate, 0}, {swap_total, 0}, {swap_used, 0}]}, {interesting_stats, []}, {cluster_compatibility_version, 1}, {version, [{kernel, "2.14.3"}, {mnesia, "4.4.17"}, {stdlib, "1.17.3"}, {os_mon, "2.2.5"}, {ns_server, "1.7.1r-42-gd713cca"}, {sasl, "2.1.9.3"}]}, {system_arch, "x86_64-unknown-linux-gnu"}, {wall_clock, 233}, {memory_data, {644874240, 633655296, {<5901.162.0>, 371936}}}, {disk_data, [{"/", 5904748, 66}, {"/dev/shm", 314880, 0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 10020 kB\nBuffers: 20288 kB\nCached: 428068 kB\nSwapCached: 0 kB\nActive: 217736 kB\nInactive: 325396 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 10020 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 1028 kB\nWriteback: 0 kB\nAnonPages: 94828 kB\nMapped: 11500 kB\nSlab: 36276 kB\nPageTables: 3896 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 538212 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory, 644874240}, {free_swap, 0}, {total_swap, 0}, {cached_memory, 438341632}, {buffered_memory, 20774912}, {free_memory, 10260480}, {total_memory, 644874240}]}, {statistics, [{wall_clock, {228849, 1}}, {context_switches, {31569, 0}}, {garbage_collection, {5995, 9857662, 0}}, {io, {{input, 7196493}, {output, 2054018}}}, {reductions, {3692486, 173663}}, {run_queue, 0}, {runtime, {270, 0}}]}]}] INFO REPORT <5901.118.0> 2011-07-13 23:53:29 =============================================================================== config change: {node,'ns_1@10.88.165.188',moxi} -> [{port,11211},{verbosity,[]}] INFO REPORT <5901.118.0> 2011-07-13 23:53:29 =============================================================================== config change: {node,'ns_1@10.88.165.188',memcached} -> [{port,11210}, {dbdir,"/opt/membase/var/lib/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"/opt/membase/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"/opt/membase/lib/memcached/ep.so"}, {initfile,"/opt/membase/etc/membase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine,"/opt/membase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}] INFO REPORT <5901.118.0> 2011-07-13 23:53:29 =============================================================================== config change: {node,'ns_1@10.88.165.188',membership} -> active INFO REPORT <5901.118.0> 2011-07-13 23:53:29 =============================================================================== config change: {node,'ns_1@10.88.165.188',isasl} -> [{path,"/opt/membase/var/lib/membase/data/isasl.pw"}] INFO REPORT <5901.118.0> 2011-07-13 23:53:29 =============================================================================== ns_node_disco_conf_events config on nodes_wanted INFO REPORT <5901.118.0> 2011-07-13 23:53:29 =============================================================================== config change: nodes_wanted -> ['ns_1@10.88.165.188'] INFO REPORT <5901.144.0> 2011-07-13 23:53:29 =============================================================================== ns_1@10.88.165.188:<5901.144.0>:mb_master:329: List of peers has changed from ['ns_1@127.0.0.1'] to ['ns_1@10.88.165.188'] INFO REPORT <5901.852.0> 2011-07-13 23:53:29 =============================================================================== ns_node_disco cookie_sync INFO REPORT <5901.852.0> 2011-07-13 23:53:29 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.88.165.188'], with cookie: bhxehmsjnydvwwbh INFO REPORT <5901.118.0> 2011-07-13 23:53:29 =============================================================================== config change: {node,'ns_1@10.88.165.188',config_version} -> {1,7,1} INFO REPORT <5901.852.0> 2011-07-13 23:53:29 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.88.165.188'], with cookie: bhxehmsjnydvwwbh INFO REPORT <5901.118.0> 2011-07-13 23:53:29 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <5901.137.0> 2011-07-13 23:53:29 =============================================================================== Pushing config INFO REPORT <5901.137.0> 2011-07-13 23:53:29 =============================================================================== Pushing config done INFO REPORT <5901.63.0> 2011-07-13 23:53:29 =============================================================================== ns_1@10.88.165.188:<5901.63.0>:ns_cluster:102: handling complete_join([{<<"targetNode">>, <<"ns_1@10.88.165.188">>}, {<<"availableStorage">>, {struct, [{<<"hdd">>, [{struct, [{<<"path">>, <<"/">>}, {<<"sizeKBytes">>, 5904748}, {<<"usagePercent">>, 66}]}, {struct, [{<<"path">>, <<"/dev/shm">>}, {<<"sizeKBytes">>, 314880}, {<<"usagePercent">>, 0}]}]}]}}, {<<"memoryQuota">>, 492}, {<<"storageTotals">>, {struct, [{<<"ram">>, {struct, [{<<"usedByData">>, 0}, {<<"total">>, 644874240}, {<<"quotaTotal">>, 515899392}, {<<"used">>, 633511936}]}}, {<<"hdd">>, {struct, [{<<"usedByData">>, 0}, {<<"total">>, 6046461952.0}, {<<"quotaTotal">>, 6046461952.0}, {<<"used">>, 3990664888.0}, {<<"free">>, 2055797064}]}}]}}, {<<"storage">>, {struct, [{<<"ssd">>, []}, {<<"hdd">>, [{struct, [{<<"path">>, <<"/opt/membase/var/lib/membase/data">>}, {<<"quotaMb">>, <<"none">>}, {<<"state">>, <<"ok">>}]}]}]}}, {<<"systemStats">>, {struct, [{<<"cpu_utilization_rate">>, 8.080808080808081}, {<<"swap_total">>, 0}, {<<"swap_used">>, 0}]}}, {<<"interestingStats">>, {struct, []}}, {<<"uptime">>, <<"332">>}, {<<"memoryTotal">>, 644874240}, {<<"memoryFree">>, 11362304}, {<<"mcdMemoryReserved">>, 492}, {<<"mcdMemoryAllocated">>, 492}, {<<"otpNode">>, <<"ns_1@10.88.23.62">>}, {<<"otpCookie">>, <<"brohpzslpcmrmtnv">>}, {<<"clusterMembership">>, <<"active">>}, {<<"status">>, <<"healthy">>}, {<<"hostname">>, <<"10.88.23.62:8091">>}, {<<"clusterCompatibility">>, 1}, {<<"version">>, <<"1.7.1r-42-gd713cca">>}, {<<"os">>, <<"x86_64-unknown-linux-gnu">>}, {<<"ports">>, {struct, [{<<"proxy">>, 11211}, {<<"direct">>, 11210}]}}]) INFO REPORT <5901.63.0> 2011-07-13 23:53:29 =============================================================================== ns_log: logging ns_cluster:2:Node 'ns_1@10.88.165.188' is joining cluster via node 'ns_1@10.88.23.62'. INFO REPORT <5901.118.0> 2011-07-13 23:53:29 =============================================================================== config change: i_am_a_dead_man -> true INFO REPORT <5901.234.0> 2011-07-13 23:53:29 =============================================================================== menelaus_web streaming socket closed by client INFO REPORT <5901.237.0> 2011-07-13 23:53:29 =============================================================================== moxi<0.237.0>: EOL on stdin. Exiting ERROR REPORT <5901.884.0> 2011-07-13 23:53:29 =============================================================================== ** Connection attempt from disallowed node 'ns_1@10.88.23.62' ** INFO REPORT <5901.144.0> 2011-07-13 23:53:30 =============================================================================== ns_1@10.88.165.188:<5901.144.0>:mb_master:135: Node changed name from 'ns_1@127.0.0.1' to 'ns_1@10.88.165.188'. Updating state. ERROR REPORT <5901.153.0> 2011-07-13 23:53:30 =============================================================================== ns_1@10.88.165.188:<5901.153.0>:ns_heart:137: Failed to grab system stats: {'EXIT',{noproc,{gen_server,call, [{'stats_reader-@system','ns_1@10.88.165.188'}, {latest,"minute"}]}}} INFO REPORT <5901.167.0> 2011-07-13 23:53:30 =============================================================================== memcached<0.167.0>: EOL on stdin. Exiting ERROR REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== ** gen_event handler ns_port_init crashed. ** Was installed in ns_config_events ** Last event was: [{i_am_a_dead_man,true}] ** When handler state == {state} ** Reason == {shutdown,{gen_server,call, [ns_port_sup,which_children,infinity]}} INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <5901.137.0> 2011-07-13 23:53:30 =============================================================================== Pushing config INFO REPORT <5901.137.0> 2011-07-13 23:53:30 =============================================================================== Pushing config done INFO REPORT <5901.63.0> 2011-07-13 23:53:30 =============================================================================== ns_cluster: joining cluster. Child has exited. INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: otp -> [{cookie,brohpzslpcmrmtnv}] INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: nodes_wanted -> ['ns_1@10.88.165.188','ns_1@10.88.23.62'] INFO REPORT <5901.63.0> 2011-07-13 23:53:30 =============================================================================== pre-join cleaned config is: {config,{}, [[], [{directory,"/opt/membase/var/lib/membase/config"}, {nodes_wanted,['ns_1@127.0.0.1']}, {{node,'ns_1@127.0.0.1',membership},active}, {{node,'ns_1@127.0.0.1',rest},[{port,8091}]}, {rest_creds,[{creds,[]}]}, {{node,'ns_1@127.0.0.1',isasl}, [{path,"/opt/membase/var/lib/membase/data/isasl.pw"}]}, {{node,'ns_1@127.0.0.1',memcached}, [{port,11210}, {dbdir,"/opt/membase/var/lib/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"/opt/membase/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"/opt/membase/lib/memcached/ep.so"}, {initfile,"/opt/membase/etc/membase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine,"/opt/membase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}]}, {memory_quota,492}, {buckets,[{configs,[]}]}, {{node,'ns_1@127.0.0.1',moxi},[{port,11211},{verbosity,[]}]}, {port_servers, [{moxi,"/opt/membase/bin/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{rest,port}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR", {"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD", {"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,exit_status,port_server_send_eol,stderr_to_stdout, stream]}, {memcached,"/opt/membase/bin/memcached", ["-X","/opt/membase/lib/memcached/stdin_term_handler.so","-p", {"~B",[port]}, "-E","/opt/membase/lib/memcached/bucket_engine.so","-B", "binary","-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status,port_server_send_eol, stream]}]}, {{node,'ns_1@127.0.0.1',ns_log}, [{filename,"/opt/membase/var/lib/membase/data/ns_log"}]}, {email_alerts, [{recipients,["root@localhost"]}, {sender,"membase@localhost"}, {enabled,true}, {email_server, [{user,[]}, {pass,[]}, {host,"localhost"}, {port,25}, {encrypt,false}]}, {alerts, [auto_failover_node,auto_failover_maximum_reached, auto_failover_other_nodes_down, auto_failover_cluster_too_small]}]}, {replication,[{enabled,true}]}, {auto_failover_cfg, [{enabled,false},{timeout,30},{max_nodes,1},{count,0}]}]], [[{nodes_wanted,['ns_1@10.88.165.188','ns_1@10.88.23.62']}, {otp, [{'_vclock',[{'ns_1@10.88.165.188',{1,63477834810}}]}, {cookie,brohpzslpcmrmtnv}]}, {directory,"/opt/membase/var/lib/membase/config"}, {{node,'ns_1@10.88.165.188',config_version}, [{'_vclock',[{'ns_1@10.88.165.188',{1,63477834809}}]}|{1,7,1}]}, {{node,'ns_1@10.88.165.188',isasl}, [{'_vclock',[{'ns_1@10.88.165.188',{1,63477834809}}]}, {path,"/opt/membase/var/lib/membase/data/isasl.pw"}]}, {{node,'ns_1@10.88.165.188',memcached}, [{'_vclock',[{'ns_1@10.88.165.188',{1,63477834809}}]}, {port,11210}, {dbdir,"/opt/membase/var/lib/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"/opt/membase/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"/opt/membase/lib/memcached/ep.so"}, {initfile,"/opt/membase/etc/membase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine,"/opt/membase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}]}, {{node,'ns_1@10.88.165.188',moxi}, [{'_vclock',[{'ns_1@10.88.165.188',{1,63477834809}}]}, {port,11211}, {verbosity,[]}]}, {{node,'ns_1@10.88.165.188',ns_log}, [{'_vclock',[{'ns_1@10.88.165.188',{1,63477834809}}]}, {filename,"/opt/membase/var/lib/membase/data/ns_log"}]}, {{node,'ns_1@10.88.165.188',rest}, [{'_vclock',[{'ns_1@10.88.165.188',{1,63477834809}}]},{port,8091}]}]], ns_config_default} INFO REPORT <5901.63.0> 2011-07-13 23:53:30 =============================================================================== ns_1@10.88.165.188:<5901.63.0>:ns_cluster:573: Connection from 'ns_1@10.88.165.188' to 'ns_1@10.88.23.62': true INFO REPORT <5901.63.0> 2011-07-13 23:53:30 =============================================================================== ns_1@10.88.165.188:<5901.63.0>:ns_cluster:582: Join status: {ok,ok}, starting ns_server_cluster back INFO REPORT <5901.910.0> 2011-07-13 23:53:30 =============================================================================== ns_log:init(): Couldn't load logs from "/opt/membase/var/lib/membase/data/ns_log": {error, enoent} PROGRESS REPORT <5901.909.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.910.0>}, {name,ns_log}, {mfargs,{ns_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.909.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.911.0>}, {name,ns_log_events}, {mfargs,{gen_event,start_link,[{local,ns_log_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.912.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<5901.913.0>}, {name,ns_node_disco_events}, {mfargs,{gen_event,start_link,[{local,ns_node_disco_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.914.0> 2011-07-13 23:53:30 =============================================================================== Initting ns_node_disco with ['ns_1@10.88.23.62'] INFO REPORT <5901.915.0> 2011-07-13 23:53:30 =============================================================================== ns_node_disco cookie_sync INFO REPORT <5901.915.0> 2011-07-13 23:53:30 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.88.165.188','ns_1@10.88.23.62'], with cookie: brohpzslpcmrmtnv INFO REPORT <5901.915.0> 2011-07-13 23:53:30 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.88.165.188','ns_1@10.88.23.62'], with cookie: brohpzslpcmrmtnv PROGRESS REPORT <5901.912.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<5901.914.0>}, {name,ns_node_disco}, {mfargs,{ns_node_disco,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.912.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<5901.917.0>}, {name,ns_node_disco_log}, {mfargs,{ns_node_disco_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.912.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<5901.918.0>}, {name,ns_node_disco_conf_events}, {mfargs,{ns_node_disco_conf_events,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.913.0> 2011-07-13 23:53:30 =============================================================================== ns_node_disco_log: nodes changed: ['ns_1@10.88.165.188','ns_1@10.88.23.62'] INFO REPORT <5901.919.0> 2011-07-13 23:53:30 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:56: init pulling INFO REPORT <5901.919.0> 2011-07-13 23:53:30 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: auto_failover_cfg -> [{enabled,false},{timeout,30},{max_nodes,1},{count,0}] INFO REPORT <5901.919.0> 2011-07-13 23:53:30 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:58: init pushing INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: buckets -> [{configs,[]}] INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: email_alerts -> [{recipients,["root@localhost"]}, {sender,"membase@localhost"}, {enabled,true}, {email_server,[{user,[]}, {pass,[]}, {host,"localhost"}, {port,25}, {encrypt,false}]}, {alerts,[auto_failover_node,auto_failover_maximum_reached, auto_failover_other_nodes_down,auto_failover_cluster_too_small]}] INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: memory_quota -> 492 INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== ns_node_disco_conf_events config on nodes_wanted INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: nodes_wanted -> ['ns_1@10.88.165.188','ns_1@10.88.23.62'] INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: port_servers -> [{moxi,"/opt/membase/bin/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{rest,port}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",{"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD",{"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,exit_status,port_server_send_eol,stderr_to_stdout,stream]}, {memcached,"/opt/membase/bin/memcached", ["-X","/opt/membase/lib/memcached/stdin_term_handler.so","-p", {"~B",[port]}, "-E","/opt/membase/lib/memcached/bucket_engine.so","-B","binary", "-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status,port_server_send_eol, stream]}] INFO REPORT <5901.924.0> 2011-07-13 23:53:30 =============================================================================== ns_node_disco cookie_sync INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: replication -> [{enabled,true}] INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: rest -> [{port,8091}] INFO REPORT <5901.919.0> 2011-07-13 23:53:30 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:62: init reannouncing INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: rest_creds -> ******** INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: uuid -> <<"78161b79-ffa7-4001-15e8-0cc7000003af">> INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: {node,'ns_1@10.88.165.188',membership} -> inactiveAdded INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: {node,'ns_1@10.88.23.62',config_version} -> {1,7,1} INFO REPORT <5901.924.0> 2011-07-13 23:53:30 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.88.165.188','ns_1@10.88.23.62'], with cookie: brohpzslpcmrmtnv INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: {node,'ns_1@10.88.23.62',isasl} -> [{path,"/opt/membase/var/lib/membase/data/isasl.pw"}] INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: {node,'ns_1@10.88.23.62',membership} -> active INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: {node,'ns_1@10.88.23.62',memcached} -> [{port,11210}, {dbdir,"/opt/membase/var/lib/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"/opt/membase/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"/opt/membase/lib/memcached/ep.so"}, {initfile,"/opt/membase/etc/membase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine,"/opt/membase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}] INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: {node,'ns_1@10.88.23.62',moxi} -> [{port,11211},{verbosity,[]}] PROGRESS REPORT <5901.912.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<5901.919.0>}, {name,ns_config_rep}, {mfargs,{ns_config_rep,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: {node,'ns_1@10.88.23.62',ns_log} -> [{filename,"/opt/membase/var/lib/membase/data/ns_log"}] PROGRESS REPORT <5901.909.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.912.0>}, {name,ns_node_disco_sup}, {mfargs,{ns_node_disco_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: {node,'ns_1@10.88.23.62',rest} -> [{port,8091}] INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== ns_node_disco_conf_events config all PROGRESS REPORT <5901.909.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.926.0>}, {name,ns_tick_event}, {mfargs,{gen_event,start_link,[{local,ns_tick_event}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.919.0> 2011-07-13 23:53:30 =============================================================================== Pushing config INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: auto_failover_cfg -> [{enabled,false},{timeout,30},{max_nodes,1},{count,0}] INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: buckets -> [{configs,[]}] INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: directory -> "/opt/membase/var/lib/membase/config" INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: email_alerts -> [{recipients,["root@localhost"]}, {sender,"membase@localhost"}, {enabled,true}, {email_server,[{user,[]}, {pass,[]}, {host,"localhost"}, {port,25}, {encrypt,false}]}, {alerts,[auto_failover_node,auto_failover_maximum_reached, auto_failover_other_nodes_down,auto_failover_cluster_too_small]}] INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: memory_quota -> 492 INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== ns_node_disco_conf_events config on nodes_wanted INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: nodes_wanted -> ['ns_1@10.88.165.188','ns_1@10.88.23.62'] INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== ns_node_disco_conf_events config on otp INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: otp -> [{cookie,brohpzslpcmrmtnv}] INFO REPORT <5901.919.0> 2011-07-13 23:53:30 =============================================================================== Pushing config done INFO REPORT <5901.930.0> 2011-07-13 23:53:30 =============================================================================== ns_node_disco cookie_sync INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: port_servers -> [{moxi,"/opt/membase/bin/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{rest,port}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",{"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD",{"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,exit_status,port_server_send_eol,stderr_to_stdout,stream]}, {memcached,"/opt/membase/bin/memcached", ["-X","/opt/membase/lib/memcached/stdin_term_handler.so","-p", {"~B",[port]}, "-E","/opt/membase/lib/memcached/bucket_engine.so","-B","binary", "-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status,port_server_send_eol, stream]}] INFO REPORT <5901.931.0> 2011-07-13 23:53:30 =============================================================================== ns_node_disco cookie_sync INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: replication -> [{enabled,true}] INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: rest -> [{port,8091}] INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: rest_creds -> ******** INFO REPORT <5901.930.0> 2011-07-13 23:53:30 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.88.165.188','ns_1@10.88.23.62'], with cookie: brohpzslpcmrmtnv INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: uuid -> <<"78161b79-ffa7-4001-15e8-0cc7000003af">> INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: {node,'ns_1@10.88.165.188',config_version} -> {1,7,1} INFO REPORT <5901.931.0> 2011-07-13 23:53:30 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.88.165.188','ns_1@10.88.23.62'], with cookie: brohpzslpcmrmtnv INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: {node,'ns_1@10.88.165.188',isasl} -> [{path,"/opt/membase/var/lib/membase/data/isasl.pw"}] INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: {node,'ns_1@10.88.165.188',membership} -> inactiveAdded INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: {node,'ns_1@10.88.165.188',memcached} -> [{port,11210}, {dbdir,"/opt/membase/var/lib/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"/opt/membase/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"/opt/membase/lib/memcached/ep.so"}, {initfile,"/opt/membase/etc/membase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine,"/opt/membase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}] INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: {node,'ns_1@10.88.165.188',moxi} -> [{port,11211},{verbosity,[]}] INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: {node,'ns_1@10.88.165.188',ns_log} -> [{filename,"/opt/membase/var/lib/membase/data/ns_log"}] INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: {node,'ns_1@10.88.165.188',rest} -> [{port,8091}] INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: {node,'ns_1@10.88.23.62',config_version} -> {1,7,1} INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: {node,'ns_1@10.88.23.62',isasl} -> [{path,"/opt/membase/var/lib/membase/data/isasl.pw"}] INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: {node,'ns_1@10.88.23.62',membership} -> active INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: {node,'ns_1@10.88.23.62',memcached} -> [{port,11210}, {dbdir,"/opt/membase/var/lib/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"/opt/membase/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"/opt/membase/lib/memcached/ep.so"}, {initfile,"/opt/membase/etc/membase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine,"/opt/membase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}] INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: {node,'ns_1@10.88.23.62',moxi} -> [{port,11211},{verbosity,[]}] INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: {node,'ns_1@10.88.23.62',ns_log} -> [{filename,"/opt/membase/var/lib/membase/data/ns_log"}] INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== config change: {node,'ns_1@10.88.23.62',rest} -> [{port,8091}] INFO REPORT <5901.118.0> 2011-07-13 23:53:30 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <5901.919.0> 2011-07-13 23:53:30 =============================================================================== Pushing config INFO REPORT <5901.927.0> 2011-07-13 23:53:30 =============================================================================== ns_1@10.88.165.188:<5901.927.0>:mb_master:94: Starting as candidate. Peers: ['ns_1@10.88.165.188', 'ns_1@10.88.23.62'] INFO REPORT <5901.919.0> 2011-07-13 23:53:30 =============================================================================== Pushing config done PROGRESS REPORT <5901.909.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.927.0>}, {name,mb_master}, {mfargs,{mb_master,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.909.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.934.0>}, {name,buckets_events}, {mfargs,{gen_event,start_link,[{local,buckets_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.935.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,ns_mail_sup} started [{pid,<5901.936.0>}, {name,ns_mail}, {mfargs,{ns_mail,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.935.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,ns_mail_sup} started [{pid,<5901.937.0>}, {name,ns_mail_log}, {mfargs,{ns_mail_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.909.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.935.0>}, {name,ns_mail_sup}, {mfargs,{ns_mail_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <5901.911.0> 2011-07-13 23:53:30 =============================================================================== ns_1@10.88.165.188:<5901.911.0>:ns_mail_log:45: ns_mail_log started up PROGRESS REPORT <5901.909.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.938.0>}, {name,ns_heart}, {mfargs,{ns_heart,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.909.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.939.0>}, {name,ns_doctor}, {mfargs,{ns_doctor,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.941.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<5901.942.0>}, {name,menelaus_web}, {mfargs,{menelaus_web,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <5901.941.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<5901.944.0>}, {name,menelaus_event}, {mfargs,{menelaus_event,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <5901.941.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<5901.945.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <5901.941.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<5901.946.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <5901.909.0> 2011-07-13 23:53:30 =============================================================================== ns_log: logging menelaus_sup:1:Membase Server has started on web port 8091 on node 'ns_1@10.88.165.188'. PROGRESS REPORT <5901.909.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.941.0>}, {name,menelaus}, {mfargs,{menelaus_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.947.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<5901.948.0>}, {name,ns_port_init}, {mfargs,{ns_port_init,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <5901.949.0> 2011-07-13 23:53:30 =============================================================================== starting ns_port_server with delay of 5000 PROGRESS REPORT <5901.947.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<5901.949.0>}, {name, {moxi,"/opt/membase/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]}}, {mfargs, {supervisor_cushion,start_link, [moxi,5000,ns_port_server,start_link, [moxi,"/opt/membase/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]]]}}, {restart_type,permanent}, {shutdown,10000}, {child_type,worker}] INFO REPORT <5901.951.0> 2011-07-13 23:53:30 =============================================================================== starting ns_port_server with delay of 5000 PROGRESS REPORT <5901.947.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<5901.951.0>}, {name, {memcached,"/opt/membase/bin/memcached", ["-X","/opt/membase/lib/memcached/stdin_term_handler.so", "-p","11210","-E", "/opt/membase/lib/memcached/bucket_engine.so","-B", "binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE", "/opt/membase/var/lib/membase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status, port_server_send_eol,stream]}}, {mfargs, {supervisor_cushion,start_link, [memcached,5000,ns_port_server,start_link, [memcached,"/opt/membase/bin/memcached", ["-X","/opt/membase/lib/memcached/stdin_term_handler.so", "-p","11210","-E", "/opt/membase/lib/memcached/bucket_engine.so","-B", "binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE", "/opt/membase/var/lib/membase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status, port_server_send_eol,stream]]]}}, {restart_type,permanent}, {shutdown,10000}, {child_type,worker}] PROGRESS REPORT <5901.909.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.947.0>}, {name,ns_port_sup}, {mfargs,{ns_port_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,60000}, {child_type,worker}] PROGRESS REPORT <5901.909.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.953.0>}, {name,ns_stats_event}, {mfargs,{gen_event,start_link,[{local,ns_stats_event}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.909.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.954.0>}, {name,ns_bucket_worker}, {mfargs,{work_queue,start_link,[ns_bucket_worker]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.909.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.955.0>}, {name,ns_bucket_sup}, {mfargs,{ns_bucket_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.909.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.956.0>}, {name,system_stats_collector}, {mfargs,{system_stats_collector,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.909.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.958.0>}, {name,{stats_archiver,"@system"}}, {mfargs,{stats_archiver,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.924.0> 2011-07-13 23:53:30 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.88.165.188','ns_1@10.88.23.62'], with cookie: brohpzslpcmrmtnv PROGRESS REPORT <5901.909.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.959.0>}, {name,{stats_reader,"@system"}}, {mfargs,{stats_reader,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.930.0> 2011-07-13 23:53:30 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.88.165.188','ns_1@10.88.23.62'], with cookie: brohpzslpcmrmtnv INFO REPORT <5901.931.0> 2011-07-13 23:53:30 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.88.165.188','ns_1@10.88.23.62'], with cookie: brohpzslpcmrmtnv PROGRESS REPORT <5901.909.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.960.0>}, {name,ns_moxi_sup_work_queue}, {mfargs,{work_queue,start_link,[ns_moxi_sup_work_queue]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.909.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.961.0>}, {name,ns_moxi_sup}, {mfargs,{ns_moxi_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.53.0> 2011-07-13 23:53:30 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<5901.909.0>}, {name,ns_server_sup}, {mfargs,{ns_server_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <5901.63.0> 2011-07-13 23:53:30 =============================================================================== ns_log: logging ns_cluster:3:Node ns_1@10.88.165.188 joined cluster INFO REPORT <5901.63.0> 2011-07-13 23:53:30 =============================================================================== ns_1@10.88.165.188:<5901.63.0>:ns_cluster:104: complete_join([{<<"targetNode">>, <<"ns_1@10.88.165.188">>}, {<<"availableStorage">>, {struct, [{<<"hdd">>, [{struct, [{<<"path">>, <<"/">>}, {<<"sizeKBytes">>, 5904748}, {<<"usagePercent">>, 66}]}, {struct, [{<<"path">>, <<"/dev/shm">>}, {<<"sizeKBytes">>, 314880}, {<<"usagePercent">>, 0}]}]}]}}, {<<"memoryQuota">>, 492}, {<<"storageTotals">>, {struct, [{<<"ram">>, {struct, [{<<"usedByData">>, 0}, {<<"total">>, 644874240}, {<<"quotaTotal">>, 515899392}, {<<"used">>, 633511936}]}}, {<<"hdd">>, {struct, [{<<"usedByData">>, 0}, {<<"total">>, 6046461952.0}, {<<"quotaTotal">>, 6046461952.0}, {<<"used">>, 3990664888.0}, {<<"free">>, 2055797064}]}}]}}, {<<"storage">>, {struct, [{<<"ssd">>, []}, {<<"hdd">>, [{struct, [{<<"path">>, <<"/opt/membase/var/lib/membase/data">>}, {<<"quotaMb">>, <<"none">>}, {<<"state">>, <<"ok">>}]}]}]}}, {<<"systemStats">>, {struct, [{<<"cpu_utilization_rate">>, 8.080808080808081}, {<<"swap_total">>, 0}, {<<"swap_used">>, 0}]}}, {<<"interestingStats">>, {struct,[]}}, {<<"uptime">>, <<"332">>}, {<<"memoryTotal">>, 644874240}, {<<"memoryFree">>, 11362304}, {<<"mcdMemoryReserved">>, 492}, {<<"mcdMemoryAllocated">>, 492}, {<<"otpNode">>, <<"ns_1@10.88.23.62">>}, {<<"otpCookie">>, <<"brohpzslpcmrmtnv">>}, {<<"clusterMembership">>, <<"active">>}, {<<"status">>, <<"healthy">>}, {<<"hostname">>, <<"10.88.23.62:8091">>}, {<<"clusterCompatibility">>, 1}, {<<"version">>, <<"1.7.1r-42-gd713cca">>}, {<<"os">>, <<"x86_64-unknown-linux-gnu">>}, {<<"ports">>, {struct, [{<<"proxy">>, 11211}, {<<"direct">>, 11210}]}}]) -> {ok, ok} INFO REPORT <5901.939.0> 2011-07-13 23:53:30 =============================================================================== ns_1@10.88.165.188:<5901.939.0>:ns_doctor:82: Got initial status [{'ns_1@10.88.165.188', [{last_heard, {1310, 615610, 511149}}, {active_buckets, []}, {ready_buckets, []}, {replication, []}, {memory, [{total, 22981288}, {processes, 4756016}, {processes_used, 4740096}, {system, 18225272}, {atom, 786201}, {atom_used, 767822}, {binary, 90680}, {code, 7424056}, {ets, 626640}]}, {system_stats, [{cpu_utilization_rate, 0.0}, {swap_total, 0}, {swap_used, 0}]}, {interesting_stats, []}, {cluster_compatibility_version, 1}, {version, [{kernel, "2.14.3"}, {mnesia, "4.4.17"}, {stdlib, "1.17.3"}, {os_mon, "2.2.5"}, {ns_server, "1.7.1r-42-gd713cca"}, {sasl, "2.1.9.3"}]}, {system_arch, "x86_64-unknown-linux-gnu"}, {wall_clock, 234}, {memory_data, {644874240, 633655296, {<5901.162.0>, 371936}}}, {disk_data, [{"/", 5904748, 66}, {"/dev/shm", 314880, 0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 10084 kB\nBuffers: 20488 kB\nCached: 427888 kB\nSwapCached: 0 kB\nActive: 218348 kB\nInactive: 325364 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 10084 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 1152 kB\nWriteback: 0 kB\nAnonPages: 95324 kB\nMapped: 11148 kB\nSlab: 36312 kB\nPageTables: 3792 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 457076 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory, 644874240}, {free_swap, 0}, {total_swap, 0}, {cached_memory, 438157312}, {buffered_memory, 20979712}, {free_memory, 10629120}, {total_memory, 644874240}]}, {statistics, [{wall_clock, {234042, 36}}, {context_switches, {35892, 0}}, {garbage_collection, {7126, 11417521, 0}}, {io, {{input, 7504793}, {output, 2448321}}}, {reductions, {4104144, 41268}}, {run_queue, 1}, {runtime, {300, 0}}]}]}, {'ns_1@10.88.23.62', [{last_heard, {1310, 615610, 511158}}, {active_buckets, []}, {ready_buckets, []}, {replication, []}, {memory, [{total, 25210288}, {processes, 6087960}, {processes_used, 6072272}, {system, 19122328}, {atom, 822673}, {atom_used, 815256}, {binary, 175816}, {code, 8081848}, {ets, 684496}]}, {system_stats, [{cpu_utilization_rate, 0.0}, {swap_total, 0}, {swap_used, 0}]}, {interesting_stats, []}, {cluster_compatibility_version, 1}, {version, [{kernel, "2.14.3"}, {mnesia, "4.4.17"}, {stdlib, "1.17.3"}, {os_mon, "2.2.5"}, {inets, "5.5.2"}, {ns_server, "1.7.1r-42-gd713cca"}, {sasl, "2.1.9.3"}]}, {system_arch, "x86_64-unknown-linux-gnu"}, {wall_clock, 333}, {memory_data, {644874240, 633511936, {<5990.234.0>, 372072}}}, {disk_data, [{"/", 5904748, 66}, {"/dev/shm", 314880, 0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 8292 kB\nBuffers: 20456 kB\nCached: 427916 kB\nSwapCached: 0 kB\nActive: 221712 kB\nInactive: 323464 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 8292 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 368 kB\nWriteback: 0 kB\nAnonPages: 96836 kB\nMapped: 11500 kB\nSlab: 36396 kB\nPageTables: 3900 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 540264 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory, 644874240}, {free_swap, 0}, {total_swap, 0}, {cached_memory, 438185984}, {buffered_memory, 20946944}, {free_memory, 8491008}, {total_memory, 644874240}]}, {statistics, [{wall_clock, {332499, 1}}, {context_switches, {49495, 0}}, {garbage_collection, {9713, 16992002, 0}}, {io, {{input, 9195970}, {output, 3967210}}}, {reductions, {6064069, 1221357}}, {run_queue, 0}, {runtime, {430, 70}}]}]}] INFO REPORT <5901.927.0> 2011-07-13 23:53:31 =============================================================================== ns_1@10.88.165.188:<5901.927.0>:mb_master:205: Changing master from undefined to 'ns_1@10.88.23.62' INFO REPORT <5901.950.0> 2011-07-13 23:53:31 =============================================================================== moxi<0.950.0>: 2011-07-13 23:53:30: (cproxy_config.c.317) env: MOXI_SASL_PLAIN_USR (13) moxi<0.950.0>: 2011-07-13 23:53:30: (cproxy_config.c.326) env: MOXI_SASL_PLAIN_PWD (8) INFO REPORT <5901.118.0> 2011-07-13 23:53:31 =============================================================================== config change: {node,'ns_1@10.88.165.188',membership} -> active INFO REPORT <5901.118.0> 2011-07-13 23:53:31 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <5901.919.0> 2011-07-13 23:53:31 =============================================================================== Pushing config INFO REPORT <5901.919.0> 2011-07-13 23:53:31 =============================================================================== Pushing config done INFO REPORT <5901.118.0> 2011-07-13 23:53:31 =============================================================================== config change: rebalance_status -> none INFO REPORT <5901.118.0> 2011-07-13 23:53:31 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <5901.919.0> 2011-07-13 23:53:31 =============================================================================== Pushing config INFO REPORT <5901.919.0> 2011-07-13 23:53:31 =============================================================================== Pushing config done INFO REPORT <5901.919.0> 2011-07-13 23:53:59 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.118.0> 2011-07-13 23:54:06 =============================================================================== config change: buckets -> [{configs,[{"default", [{num_replicas,1}, {ram_quota,515899392}, {auth_type,sasl}, {sasl_password,[]}, {type,membase}, {num_vbuckets,1024}, {servers,[]}]}]}] INFO REPORT <5901.118.0> 2011-07-13 23:54:06 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <5901.919.0> 2011-07-13 23:54:06 =============================================================================== Pushing config INFO REPORT <5901.919.0> 2011-07-13 23:54:06 =============================================================================== Pushing config done INFO REPORT <5901.123.0> 2011-07-13 23:54:06 =============================================================================== Writing isasl passwd file: "/opt/membase/var/lib/membase/data/isasl.pw" INFO REPORT <5901.954.0> 2011-07-13 23:54:06 =============================================================================== ns_1@10.88.165.188:<5901.954.0>:ns_bucket_sup:75: Starting new child: {{per_bucket_sup, "default"}, {single_bucket_sup, start_link, ["default"]}, permanent, infinity, supervisor, [single_bucket_sup]} INFO REPORT <5901.118.0> 2011-07-13 23:54:06 =============================================================================== config change: buckets -> [{configs,[{"default", [{num_replicas,1}, {ram_quota,515899392}, {auth_type,sasl}, {sasl_password,[]}, {type,membase}, {num_vbuckets,1024}, {servers,['ns_1@10.88.165.188','ns_1@10.88.23.62']}]}]}] INFO REPORT <5901.118.0> 2011-07-13 23:54:06 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <5901.919.0> 2011-07-13 23:54:06 =============================================================================== Pushing config INFO REPORT <5901.919.0> 2011-07-13 23:54:06 =============================================================================== Pushing config done PROGRESS REPORT <5901.955.0> 2011-07-13 23:54:06 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<5901.1078.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <5901.1080.0> 2011-07-13 23:54:07 =============================================================================== ns_1@10.88.165.188:<5901.1080.0>:ns_memcached:504: Created bucket "default" with config string "ht_size=3079;ht_locks=5;db_shards=4;tap_noop_interval=20;max_txn_size=1000;max_size=515899392;initfile=/opt/membase/etc/membase/init.sql;tap_keepalive=300;dbname=/opt/membase/var/lib/membase/data/default-data/default;vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB;" PROGRESS REPORT <5901.1079.0> 2011-07-13 23:54:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1080.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1079.0> 2011-07-13 23:54:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1084.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.950.0> 2011-07-13 23:54:07 =============================================================================== moxi<0.950.0>: 2011-07-13 23:54:08: (agent_config.c.705) ERROR: bad JSON configuration from http://127.0.0.1:8091/pools/default/saslBucketsStreaming: Empty serverList ({ moxi<0.950.0>: "name": "default", moxi<0.950.0>: "nodeLocator": "vbucket", moxi<0.950.0>: "saslPassword": "", moxi<0.950.0>: "nodes": [], moxi<0.950.0>: "vBucketServerMap": { moxi<0.950.0>: "hashAlgorithm": "CRC", moxi<0.950.0>: "numReplicas": 1, moxi<0.950.0>: "serverList": [], moxi<0.950.0>: "vBucketMap": [] ERROR REPORT <5901.950.0> 2011-07-13 23:54:07 =============================================================================== ns_1@10.88.165.188:<5901.950.0>:ns_port_server:161: Dropped 35 log lines from moxi PROGRESS REPORT <5901.1079.0> 2011-07-13 23:54:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1085.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1079.0> 2011-07-13 23:54:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1086.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.66.0> 2011-07-13 23:54:07 =============================================================================== ns_1@10.88.165.188:<5901.66.0>:mb_mnesia:109: Creating table 'stats_archiver-default-minute' PROGRESS REPORT <5901.1079.0> 2011-07-13 23:54:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1087.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1079.0> 2011-07-13 23:54:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1092.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.66.0> 2011-07-13 23:54:07 =============================================================================== ns_1@10.88.165.188:<5901.66.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-default-minute', [{name,'stats_archiver-default-minute'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.88.165.188']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1310,615647,688931},'ns_1@10.88.165.188'}}, {version,{{2,0},[]}}]}, {tid,95,<5901.1089.0>}} INFO REPORT <5901.66.0> 2011-07-13 23:54:07 =============================================================================== ns_1@10.88.165.188:<5901.66.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-default-minute', [{name,'stats_archiver-default-minute'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.88.165.188']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1310,615647,688931},'ns_1@10.88.165.188'}}, {version,{{2,0},[]}}]}, {tid,95,<5901.1089.0>}} INFO REPORT <5901.66.0> 2011-07-13 23:54:07 =============================================================================== ns_1@10.88.165.188:<5901.66.0>:mb_mnesia:109: Creating table 'stats_archiver-default-hour' INFO REPORT <5901.66.0> 2011-07-13 23:54:07 =============================================================================== ns_1@10.88.165.188:<5901.66.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-default-hour', [{name,'stats_archiver-default-hour'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.88.165.188']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1310,615647,714551},'ns_1@10.88.165.188'}}, {version,{{2,0},[]}}]}, {tid,96,<5901.1097.0>}} INFO REPORT <5901.66.0> 2011-07-13 23:54:07 =============================================================================== ns_1@10.88.165.188:<5901.66.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-default-hour', [{name,'stats_archiver-default-hour'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.88.165.188']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1310,615647,714551},'ns_1@10.88.165.188'}}, {version,{{2,0},[]}}]}, {tid,96,<5901.1097.0>}} INFO REPORT <5901.66.0> 2011-07-13 23:54:07 =============================================================================== ns_1@10.88.165.188:<5901.66.0>:mb_mnesia:109: Creating table 'stats_archiver-default-day' INFO REPORT <5901.66.0> 2011-07-13 23:54:07 =============================================================================== ns_1@10.88.165.188:<5901.66.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-default-day', [{name,'stats_archiver-default-day'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.88.165.188']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1310,615647,736765},'ns_1@10.88.165.188'}}, {version,{{2,0},[]}}]}, {tid,97,<5901.1104.0>}} INFO REPORT <5901.66.0> 2011-07-13 23:54:07 =============================================================================== ns_1@10.88.165.188:<5901.66.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-default-day', [{name,'stats_archiver-default-day'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.88.165.188']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1310,615647,736765},'ns_1@10.88.165.188'}}, {version,{{2,0},[]}}]}, {tid,97,<5901.1104.0>}} INFO REPORT <5901.66.0> 2011-07-13 23:54:07 =============================================================================== ns_1@10.88.165.188:<5901.66.0>:mb_mnesia:109: Creating table 'stats_archiver-default-week' INFO REPORT <5901.66.0> 2011-07-13 23:54:07 =============================================================================== ns_1@10.88.165.188:<5901.66.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-default-week', [{name,'stats_archiver-default-week'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.88.165.188']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1310,615647,749910},'ns_1@10.88.165.188'}}, {version,{{2,0},[]}}]}, {tid,98,<5901.1111.0>}} INFO REPORT <5901.66.0> 2011-07-13 23:54:07 =============================================================================== ns_1@10.88.165.188:<5901.66.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-default-week', [{name,'stats_archiver-default-week'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.88.165.188']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1310,615647,749910},'ns_1@10.88.165.188'}}, {version,{{2,0},[]}}]}, {tid,98,<5901.1111.0>}} INFO REPORT <5901.66.0> 2011-07-13 23:54:07 =============================================================================== ns_1@10.88.165.188:<5901.66.0>:mb_mnesia:109: Creating table 'stats_archiver-default-month' INFO REPORT <5901.66.0> 2011-07-13 23:54:07 =============================================================================== ns_1@10.88.165.188:<5901.66.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-default-month', [{name,'stats_archiver-default-month'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.88.165.188']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1310,615647,762334},'ns_1@10.88.165.188'}}, {version,{{2,0},[]}}]}, {tid,99,<5901.1118.0>}} INFO REPORT <5901.66.0> 2011-07-13 23:54:07 =============================================================================== ns_1@10.88.165.188:<5901.66.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-default-month', [{name,'stats_archiver-default-month'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.88.165.188']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1310,615647,762334},'ns_1@10.88.165.188'}}, {version,{{2,0},[]}}]}, {tid,99,<5901.1118.0>}} INFO REPORT <5901.66.0> 2011-07-13 23:54:07 =============================================================================== ns_1@10.88.165.188:<5901.66.0>:mb_mnesia:109: Creating table 'stats_archiver-default-year' INFO REPORT <5901.66.0> 2011-07-13 23:54:07 =============================================================================== ns_1@10.88.165.188:<5901.66.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-default-year', [{name,'stats_archiver-default-year'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.88.165.188']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1310,615647,774069},'ns_1@10.88.165.188'}}, {version,{{2,0},[]}}]}, {tid,100,<5901.1125.0>}} INFO REPORT <5901.66.0> 2011-07-13 23:54:07 =============================================================================== ns_1@10.88.165.188:<5901.66.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-default-year', [{name,'stats_archiver-default-year'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.88.165.188']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1310,615647,774069},'ns_1@10.88.165.188'}}, {version,{{2,0},[]}}]}, {tid,100,<5901.1125.0>}} ERROR REPORT <5901.938.0> 2011-07-13 23:54:07 =============================================================================== ns_1@10.88.165.188:<5901.938.0>:ns_heart:156: Failed to get stats for bucket: "default": {error,{exit,{badarg,[{erlang,hd,[[]]}, {stats_reader,'-do_handle_call/3-fun-0-',2}, {mnesia_tm,non_transaction,5}, {stats_reader,do_handle_call,3}, {stats_reader,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}}} INFO REPORT <5901.1080.0> 2011-07-13 23:54:08 =============================================================================== ns_log: logging ns_memcached:1:Bucket "default" loaded on node 'ns_1@10.88.165.188' in 0 seconds. ERROR REPORT <5901.938.0> 2011-07-13 23:54:08 =============================================================================== ns_1@10.88.165.188:<5901.938.0>:ns_heart:156: Failed to get stats for bucket: "default": {error,{exit,{badarg,[{erlang,hd,[[]]}, {stats_reader,'-do_handle_call/3-fun-0-',2}, {mnesia_tm,non_transaction,5}, {stats_reader,do_handle_call,3}, {stats_reader,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}}} INFO REPORT <5901.1085.0> 2011-07-13 23:54:08 =============================================================================== ns_1@10.88.165.188:<5901.1085.0>:stats_collector:83: Stats for bucket "default": accepting_conns 1 auth_cmds 0 auth_errors 0 bucket_active_conns 1 bucket_conns 1 bytes_read 96 bytes_written 23847 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 0 connection_structures 11 curr_connections 11 curr_items 0 curr_items_tot 0 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_bg_fetched 0 ep_commit_num 0 ep_commit_time 0 ep_commit_time_total 0 ep_data_age 0 ep_data_age_highwat 0 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 0 ep_dbname /opt/membase/var/lib/membase/data/default-data/default ep_dbshards 4 ep_diskqueue_drain 0 ep_diskqueue_fill 0 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 0 ep_flush_duration_total 0 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 0 ep_io_num_write 0 ep_io_read_bytes 0 ep_io_write_bytes 0 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 0 ep_kv_size 0 ep_latency_arith_cmd 0 ep_latency_get_cmd 0 ep_latency_store_cmd 0 ep_max_data_size 515899392 ep_max_txn_size 1000 ep_mem_high_wat 386924544 ep_mem_low_wat 309539635 ep_min_data_age 0 ep_num_active_non_resident 0 ep_num_checkpoint_remover_runs0 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 640 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 0 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 0 ep_total_del_items 0 ep_total_enqueued 0 ep_total_new_items 0 ep_total_persisted 0 ep_uncommitted_items 0 ep_vb_total 0 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.0_33_ged2a54a ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 45712 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 640 pid 2654 pointer_size 64 rejected_conns 0 rusage_system 0.072004 rusage_user 0.384024 threads 4 time 1310615647 total_connections 11 uptime 39 vb_active_curr_items 0 vb_active_eject 0 vb_active_ht_memory 0 vb_active_itm_memory 0 vb_active_num 0 vb_active_num_non_resident 0 vb_active_ops_create 0 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 0 vb_active_queue_age 0 vb_active_queue_drain 0 vb_active_queue_fill 0 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_461_gf99c147 INFO REPORT <5901.952.0> 2011-07-13 23:54:08 =============================================================================== memcached<0.952.0>: Extension support isn't implemented in this version of bucket_engine INFO REPORT <5901.118.0> 2011-07-13 23:54:08 =============================================================================== config change: buckets -> [{configs,[{"default", [{num_replicas,1}, {ram_quota,515899392}, {auth_type,sasl}, {sasl_password,[]}, {type,membase}, {num_vbuckets,1024}, {servers,['ns_1@10.88.165.188','ns_1@10.88.23.62']}, {map,[['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188','ns_1@10.88.23.62'], ['ns_1@10.88.165.188'|...], [...]|...]}]}]}] INFO REPORT <5901.118.0> 2011-07-13 23:54:08 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <5901.919.0> 2011-07-13 23:54:08 =============================================================================== Pushing config INFO REPORT <5901.919.0> 2011-07-13 23:54:08 =============================================================================== Pushing config done INFO REPORT <5901.950.0> 2011-07-13 23:54:08 =============================================================================== moxi<0.950.0>: 2011-07-13 23:54:09: (agent_config.c.705) ERROR: bad JSON configuration from http://127.0.0.1:8091/pools/default/saslBucketsStreaming: Number of buckets must be a power of two > 0 and <= MAX_BUCKETS ({ moxi<0.950.0>: "name": "default", moxi<0.950.0>: "nodeLocator": "vbucket", moxi<0.950.0>: "saslPassword": "", moxi<0.950.0>: "nodes": [{ moxi<0.950.0>: "replication": 1, moxi<0.950.0>: "clusterMembership": "active", moxi<0.950.0>: "status": "warmup", moxi<0.950.0>: "hostname": "10.88.165.188:8091", moxi<0.950.0>: "clusterCompatibility": 1, ERROR REPORT <5901.950.0> 2011-07-13 23:54:08 =============================================================================== ns_1@10.88.165.188:<5901.950.0>:ns_port_server:161: Dropped 92 log lines from moxi PROGRESS REPORT <5901.1084.0> 2011-07-13 23:54:09 =============================================================================== supervisor {local,'ns_vbm_sup-default'} started [{pid,<5901.1148.0>}, {name, {child_id, [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22, 23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41, 42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60, 61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79, 80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98, 99,100,101,102,103,104,105,106,107,108,109,110,111,112,113, 114,115,116,117,118,119,120,121,122,123,124,125,126,127, 128,129,130,131,132,133,134,135,136,137,138,139,140,141, 142,143,144,145,146,147,148,149,150,151,152,153,154,155, 156,157,158,159,160,161,162,163,164,165,166,167,168,169, 170,171,172,173,174,175,176,177,178,179,180,181,182,183, 184,185,186,187,188,189,190,191,192,193,194,195,196,197, 198,199,200,201,202,203,204,205,206,207,208,209,210,211, 212,213,214,215,216,217,218,219,220,221,222,223,224,225, 226,227,228,229,230,231,232,233,234,235,236,237,238,239, 240,241,242,243,244,245,246,247,248,249,250,251,252,253, 254,255,256,257,258,259,260,261,262,263,264,265,266,267, 268,269,270,271,272,273,274,275,276,277,278,279,280,281, 282,283,284,285,286,287,288,289,290,291,292,293,294,295, 296,297,298,299,300,301,302,303,304,305,306,307,308,309, 310,311,312,313,314,315,316,317,318,319,320,321,322,323, 324,325,326,327,328,329,330,331,332,333,334,335,336,337, 338,339,340,341,342,343,344,345,346,347,348,349,350,351, 352,353,354,355,356,357,358,359,360,361,362,363,364,365, 366,367,368,369,370,371,372,373,374,375,376,377,378,379, 380,381,382,383,384,385,386,387,388,389,390,391,392,393, 394,395,396,397,398,399,400,401,402,403,404,405,406,407, 408,409,410,411,412,413,414,415,416,417,418,419,420,421, 422,423,424,425,426,427,428,429,430,431,432,433,434,435, 436,437,438,439,440,441,442,443,444,445,446,447,448,449, 450,451,452,453,454,455,456,457,458,459,460,461,462,463, 464,465,466,467,468,469,470,471,472,473,474,475,476,477, 478,479,480,481,482,483,484,485,486,487,488,489,490,491, 492,493,494,495,496,497,498,499,500,501,502,503,504,505, 506,507,508,509,510,511], 'ns_1@10.88.23.62'}}, {mfargs, {ebucketmigrator_srv,start_link, [{"10.88.165.188",11210}, {"10.88.23.62",11210}, [{username,"default"}, {password,[]}, {vbuckets, [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20, 21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37, 38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54, 55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71, 72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88, 89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104, 105,106,107,108,109,110,111,112,113,114,115,116,117, 118,119,120,121,122,123,124,125,126,127,128,129,130, 131,132,133,134,135,136,137,138,139,140,141,142,143, 144,145,146,147,148,149,150,151,152,153,154,155,156, 157,158,159,160,161,162,163,164,165,166,167,168,169, 170,171,172,173,174,175,176,177,178,179,180,181,182, 183,184,185,186,187,188,189,190,191,192,193,194,195, 196,197,198,199,200,201,202,203,204,205,206,207,208, 209,210,211,212,213,214,215,216,217,218,219,220,221, 222,223,224,225,226,227,228,229,230,231,232,233,234, 235,236,237,238,239,240,241,242,243,244,245,246,247, 248,249,250,251,252,253,254,255,256,257,258,259,260, 261,262,263,264,265,266,267,268,269,270,271,272,273, 274,275,276,277,278,279,280,281,282,283,284,285,286, 287,288,289,290,291,292,293,294,295,296,297,298,299, 300,301,302,303,304,305,306,307,308,309,310,311,312, 313,314,315,316,317,318,319,320,321,322,323,324,325, 326,327,328,329,330,331,332,333,334,335,336,337,338, 339,340,341,342,343,344,345,346,347,348,349,350,351, 352,353,354,355,356,357,358,359,360,361,362,363,364, 365,366,367,368,369,370,371,372,373,374,375,376,377, 378,379,380,381,382,383,384,385,386,387,388,389,390, 391,392,393,394,395,396,397,398,399,400,401,402,403, 404,405,406,407,408,409,410,411,412,413,414,415,416, 417,418,419,420,421,422,423,424,425,426,427,428,429, 430,431,432,433,434,435,436,437,438,439,440,441,442, 443,444,445,446,447,448,449,450,451,452,453,454,455, 456,457,458,459,460,461,462,463,464,465,466,467,468, 469,470,471,472,473,474,475,476,477,478,479,480,481, 482,483,484,485,486,487,488,489,490,491,492,493,494, 495,496,497,498,499,500,501,502,503,504,505,506,507, 508,509,510,511]}, {takeover,false}, {suffix,"ns_1@10.88.23.62"}]]}}, {restart_type,permanent}, {shutdown,60000}, {child_type,worker}] INFO REPORT <5901.1148.0> 2011-07-13 23:54:10 =============================================================================== ns_1@10.88.165.188:<5901.1148.0>:ebucketmigrator_srv:132: CheckpointIdsDict: {dict,1024,205,256,128,1025,615, {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, {{[[0|1],[311|1],[622|0],[933|0]], [[179|1],[490|1],[545|0],[856|0]], [[102|1],[413|1],[724|0],[779|0]], [[25|1],[336|1],[647|0],[958|0]], [[204|1],[259|1],[570|0],[881|0]], [[127|1],[438|1],[749|0],[804|0]], [[50|1],[361|1],[672|0],[983|0]], [[229|1],[284|1],[595|0],[906|0]], [[152|1],[463|1],[518|0],[829|0]], [[75|1],[386|1],[697|0],[1008|0]], [[254|1],[309|1],[620|0],[931|0]], [[177|1],[488|1],[543|0],[854|0]], [[100|1],[411|1],[722|0],[777|0]], [[23|1],[334|1],[645|0],[956|0]], [[202|1],[257|1],[568|0],[879|0]], [[125|1],[436|1],[747|0],[802|0]]}, {[[48|1],[359|1],[670|0],[981|0]], [[227|1],[282|1],[593|0],[904|0]], [[150|1],[461|1],[516|0],[827|0]], [[73|1],[384|1],[695|0],[1006|0]], [[252|1],[307|1],[618|0],[929|0]], [[175|1],[486|1],[541|0],[852|0]], [[98|1],[409|1],[720|0],[775|0]], [[21|1],[332|1],[643|0],[954|0]], [[200|1],[511|1],[566|0],[877|0]], [[123|1],[434|1],[745|0],[800|0]], [[46|1],[357|1],[668|0],[979|0]], [[225|1],[280|1],[591|0],[902|0]], [[148|1],[459|1],[514|0],[825|0]], [[71|1],[382|1],[693|0],[1004|0]], [[250|1],[305|1],[616|0],[927|0]], [[173|1],[484|1],[539|0],[850|0]]}, {[[96|1],[407|1],[718|0],[773|0]], [[19|1],[330|1],[641|0],[952|0]], [[198|1],[509|1],[564|0],[875|0]], [[121|1],[432|1],[743|0],[798|0]], [[44|1],[355|1],[666|0],[977|0]], [[223|1],[278|1],[589|0],[900|0]], [[146|1],[457|1],[512|0],[823|0]], [[69|1],[380|1],[691|0],[1002|0]], [[248|1],[303|1],[614|0],[925|0]], [[171|1],[482|1],[537|0],[848|0]], [[94|1],[405|1],[716|0],[771|0]], [[17|1],[328|1],[639|0],[950|0]], [[196|1],[507|1],[562|0],[873|0]], [[119|1],[430|1],[741|0],[796|0]], [[42|1],[353|1],[664|0],[975|0]], [[221|1],[276|1],[587|0],[898|0]]}, {[[144|1],[455|1],[766|0],[821|0]], [[67|1],[378|1],[689|0],[1000|0]], [[246|1],[301|1],[612|0],[923|0]], [[169|1],[480|1],[535|0],[846|0]], [[92|1],[403|1],[714|0],[769|0]], [[15|1],[326|1],[637|0],[948|0]], [[194|1],[505|1],[560|0],[871|0]], [[117|1],[428|1],[739|0],[794|0]], [[40|1],[351|1],[662|0],[973|0]], [[219|1],[274|1],[585|0],[896|0]], [[142|1],[453|1],[764|0],[819|0]], [[65|1],[376|1],[687|0],[998|0]], [[244|1],[299|1],[610|0],[921|0]], [[167|1],[478|1],[533|0],[844|0]], [[90|1],[401|1],[712|0],[1023|0]], [[13|1],[324|1],[635|0],[946|0]]}, {[[192|1],[503|1],[558|0],[869|0]], [[115|1],[426|1],[737|0],[792|0]], [[38|1],[349|1],[660|0],[971|0]], [[217|1],[272|1],[583|0],[894|0]], [[140|1],[451|1],[762|0],[817|0]], [[63|1],[374|1],[685|0],[996|0]], [[242|1],[297|1],[608|0],[919|0]], [[165|1],[476|1],[531|0],[842|0]], [[88|1],[399|1],[710|0],[1021|0]], [[11|1],[322|1],[633|0],[944|0]], [[190|1],[501|1],[556|0],[867|0]], [[113|1],[424|1],[735|0],[790|0]], [[36|1],[347|1],[658|0],[969|0]], [[87|1],[215|1],[270|1],[398|1],[581|0],[709|0],[892|0],[1020|0]], [[10|1],[138|1],[321|1],[449|1],[632|0],[760|0],[815|0],[943|0]], [[61|1],[189|1],[372|1],[500|1],[555|0],[683|0],[866|0],[994|0]]}, {[[112|1],[240|1],[295|1],[423|1],[606|0],[734|0],[789|0],[917|0]], [[35|1],[163|1],[346|1],[474|1],[529|0],[657|0],[840|0],[968|0]], [[86|1],[214|1],[269|1],[397|1],[580|0],[708|0],[891|0],[1019|0]], [[9|1],[137|1],[320|1],[448|1],[631|0],[759|0],[814|0],[942|0]], [[60|1],[188|1],[371|1],[499|1],[554|0],[682|0],[865|0],[993|0]], [[111|1],[239|1],[294|1],[422|1],[605|0],[733|0],[788|0],[916|0]], [[34|1],[162|1],[345|1],[473|1],[528|0],[656|0],[839|0],[967|0]], [[85|1],[213|1],[268|1],[396|1],[579|0],[707|0],[890|0],[1018|0]], [[8|1],[136|1],[319|1],[447|1],[630|0],[758|0],[813|0],[941|0]], [[59|1],[187|1],[370|1],[498|1],[553|0],[681|0],[864|0],[992|0]], [[110|1],[238|1],[293|1],[421|1],[604|0],[732|0],[787|0],[915|0]], [[33|1],[161|1],[344|1],[472|1],[527|0],[655|0],[838|0],[966|0]], [[84|1],[212|1],[267|1],[395|1],[578|0],[706|0],[889|0],[1017|0]], [[7|1],[135|1],[318|1],[446|1],[629|0],[757|0],[812|0],[940|0]], [[58|1],[186|1],[369|1],[497|1],[552|0],[680|0],[863|0],[991|0]], [[109|1],[237|1],[292|1],[420|1],[603|0],[731|0],[786|0],[914|0]]}, {[[32|1],[160|1],[343|1],[471|1],[526|0],[654|0],[837|0],[965|0]], [[83|1],[211|1],[266|1],[394|1],[577|0],[705|0],[888|0],[1016|0]], [[6|1],[134|1],[317|1],[445|1],[628|0],[756|0],[811|0],[939|0]], [[57|1],[185|1],[368|1],[496|1],[551|0],[679|0],[862|0],[990|0]], [[108|1],[236|1],[291|1],[419|1],[602|0],[730|0],[785|0],[913|0]], [[31|1],[159|1],[342|1],[470|1],[525|0],[653|0],[836|0],[964|0]], [[82|1],[210|1],[265|1],[393|1],[576|0],[704|0],[887|0],[1015|0]], [[5|1],[133|1],[316|1],[444|1],[627|0],[755|0],[810|0],[938|0]], [[56|1],[184|1],[367|1],[495|1],[550|0],[678|0],[861|0],[989|0]], [[107|1],[235|1],[290|1],[418|1],[601|0],[729|0],[784|0],[912|0]], [[30|1],[158|1],[341|1],[469|1],[524|0],[652|0],[835|0],[963|0]], [[81|1],[209|1],[264|1],[392|1],[575|0],[703|0],[886|0],[1014|0]], [[4|1],[132|1],[315|1],[443|1],[626|0],[754|0],[809|0],[937|0]], [[55|1],[183|1],[366|1],[494|1],[549|0],[677|0],[860|0],[988|0]], [[106|1],[234|1],[289|1],[417|1],[600|0],[728|0],[783|0],[911|0]], [[29|1],[157|1],[340|1],[468|1],[523|0],[651|0],[834|0],[962|0]]}, {[[80|1],[208|1],[263|1],[391|1],[574|0],[702|0],[885|0],[1013|0]], [[3|1],[131|1],[314|1],[442|1],[625|0],[753|0],[808|0],[936|0]], [[54|1],[182|1],[365|1],[493|1],[548|0],[676|0],[859|0],[987|0]], [[105|1],[233|1],[288|1],[416|1],[599|0],[727|0],[782|0],[910|0]], [[28|1],[156|1],[339|1],[467|1],[522|0],[650|0],[833|0],[961|0]], [[79|1],[207|1],[262|1],[390|1],[573|0],[701|0],[884|0],[1012|0]], [[2|1],[130|1],[313|1],[441|1],[624|0],[752|0],[807|0],[935|0]], [[53|1],[181|1],[364|1],[492|1],[547|0],[675|0],[858|0],[986|0]], [[104|1],[232|1],[287|1],[415|1],[598|0],[726|0],[781|0],[909|0]], [[27|1],[155|1],[338|1],[466|1],[521|0],[649|0],[832|0],[960|0]], [[78|1],[206|1],[261|1],[389|1],[572|0],[700|0],[883|0],[1011|0]], [[1|1],[129|1],[312|1],[440|1],[623|0],[751|0],[806|0],[934|0]], [[52|1],[180|1],[363|1],[491|1],[546|0],[674|0],[857|0],[985|0]], [[103|1],[231|1],[286|1],[414|1],[597|0],[725|0],[780|0],[908|0]], [[26|1],[154|1],[337|1],[465|1],[520|0],[648|0],[831|0],[959|0]], [[77|1],[205|1],[260|1],[388|1],[571|0],[699|0],[882|0],[1010|0]]}, {[[128|1],[439|1],[750|0],[805|0]], [[51|1],[362|1],[673|0],[984|0]], [[230|1],[285|1],[596|0],[907|0]], [[153|1],[464|1],[519|0],[830|0]], [[76|1],[387|1],[698|0],[1009|0]], [[255|1],[310|1],[621|0],[932|0]], [[178|1],[489|1],[544|0],[855|0]], [[101|1],[412|1],[723|0],[778|0]], [[24|1],[335|1],[646|0],[957|0]], [[203|1],[258|1],[569|0],[880|0]], [[126|1],[437|1],[748|0],[803|0]], [[49|1],[360|1],[671|0],[982|0]], [[228|1],[283|1],[594|0],[905|0]], [[151|1],[462|1],[517|0],[828|0]], [[74|1],[385|1],[696|0],[1007|0]], [[253|1],[308|1],[619|0],[930|0]]}, {[[176|1],[487|1],[542|0],[853|0]], [[99|1],[410|1],[721|0],[776|0]], [[22|1],[333|1],[644|0],[955|0]], [[201|1],[256|1],[567|0],[878|0]], [[124|1],[435|1],[746|0],[801|0]], [[47|1],[358|1],[669|0],[980|0]], [[226|1],[281|1],[592|0],[903|0]], [[149|1],[460|1],[515|0],[826|0]], [[72|1],[383|1],[694|0],[1005|0]], [[251|1],[306|1],[617|0],[928|0]], [[174|1],[485|1],[540|0],[851|0]], [[97|1],[408|1],[719|0],[774|0]], [[20|1],[331|1],[642|0],[953|0]], [[199|1],[510|1],[565|0],[876|0]], [[122|1],[433|1],[744|0],[799|0]], [[45|1],[356|1],[667|0],[978|0]]}, {[[224|1],[279|1],[590|0],[901|0]], [[147|1],[458|1],[513|0],[824|0]], [[70|1],[381|1],[692|0],[1003|0]], [[249|1],[304|1],[615|0],[926|0]], [[172|1],[483|1],[538|0],[849|0]], [[95|1],[406|1],[717|0],[772|0]], [[18|1],[329|1],[640|0],[951|0]], [[197|1],[508|1],[563|0],[874|0]], [[120|1],[431|1],[742|0],[797|0]], [[43|1],[354|1],[665|0],[976|0]], [[222|1],[277|1],[588|0],[899|0]], [[145|1],[456|1],[767|0],[822|0]], [[68|1],[379|1],[690|0],[1001|0]], [[247|1],[302|1],[613|0],[924|0]], [[170|1],[481|1],[536|0],[847|0]], [[93|1],[404|1],[715|0],[770|0]]}, {[[16|1],[327|1],[638|0],[949|0]], [[195|1],[506|1],[561|0],[872|0]], [[118|1],[429|1],[740|0],[795|0]], [[41|1],[352|1],[663|0],[974|0]], [[220|1],[275|1],[586|0],[897|0]], [[143|1],[454|1],[765|0],[820|0]], [[66|1],[377|1],[688|0],[999|0]], [[245|1],[300|1],[611|0],[922|0]], [[168|1],[479|1],[534|0],[845|0]], [[91|1],[402|1],[713|0],[768|0]], [[14|1],[325|1],[636|0],[947|0]], [[193|1],[504|1],[559|0],[870|0]], [[116|1],[427|1],[738|0],[793|0]], [[39|1],[350|1],[661|0],[972|0]], [[218|1],[273|1],[584|0],[895|0]], [[141|1],[452|1],[763|0],[818|0]]}, {[[64|1],[375|1],[686|0],[997|0]], [[243|1],[298|1],[609|0],[920|0]], [[166|1],[477|1],[532|0],[843|0]], [[89|1],[400|1],[711|0],[1022|0]], [[12|1],[323|1],[634|0],[945|0]], [[191|1],[502|1],[557|0],[868|0]], [[114|1],[425|1],[736|0],[791|0]], [[37|1],[348|1],[659|0],[970|0]], [[216|1],[271|1],[582|0],[893|0]], [[139|1],[450|1],[761|0],[816|0]], [[62|1],[373|1],[684|0],[995|0]], [[241|1],[296|1],[607|0],[918|0]], [[164|1],[475|1],[530|0],[841|0]], [],[],[]}, {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}, {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}} INFO REPORT <5901.1148.0> 2011-07-13 23:54:10 =============================================================================== ns_1@10.88.165.188:<5901.1148.0>:ebucketmigrator_srv:162: Starting tap stream: [{vbuckets,[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24, 25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46, 47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68, 69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90, 91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108, 109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124, 125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140, 141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156, 157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172, 173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188, 189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204, 205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220, 221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236, 237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252, 253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268, 269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284, 285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300, 301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316, 317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332, 333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348, 349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364, 365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380, 381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396, 397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412, 413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428, 429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444, 445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460, 461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476, 477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492, 493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508, 509,510,511]}, {checkpoints,[{0,0}, {1,0}, {2,0}, {3,0}, {4,0}, {5,0}, {6,0}, {7,0}, {8,0}, {9,0}, {10,0}, {11,0}, {12,0}, {13,0}, {14,0}, {15,0}, {16,0}, {17,0}, {18,0}, {19,0}, {20,0}, {21,0}, {22,0}, {23,0}, {24,0}, {25,0}, {26,0}, {27,0}, {28,0}, {29,0}, {30,0}, {31,0}, {32,0}, {33,0}, {34,0}, {35,0}, {36,0}, {37,0}, {38,0}, {39,0}, {40,0}, {41,0}, {42,0}, {43,0}, {44,0}, {45,0}, {46,0}, {47,0}, {48,0}, {49,0}, {50,0}, {51,0}, {52,0}, {53,0}, {54,0}, {55,0}, {56,0}, {57,0}, {58,0}, {59,0}, {60,0}, {61,0}, {62,0}, {63,0}, {64,0}, {65,0}, {66,0}, {67,0}, {68,0}, {69,0}, {70,0}, {71,0}, {72,0}, {73,0}, {74,0}, {75,0}, {76,0}, {77,0}, {78,0}, {79,0}, {80,0}, {81,0}, {82,0}, {83,0}, {84,0}, {85,0}, {86,0}, {87,0}, {88,0}, {89,0}, {90,0}, {91,0}, {92,0}, {93,0}, {94,0}, {95,0}, {96,0}, {97,0}, {98,0}, {99,0}, {100,0}, {101,0}, {102,0}, {103,0}, {104,0}, {105,0}, {106,0}, {107,0}, {108,0}, {109,0}, {110,0}, {111,0}, {112,0}, {113,0}, {114,0}, {115,0}, {116,0}, {117,0}, {118,0}, {119,0}, {120,0}, {121,0}, {122,0}, {123,0}, {124,0}, {125,0}, {126,0}, {127,0}, {128,0}, {129,0}, {130,0}, {131,0}, {132,0}, {133,0}, {134,0}, {135,0}, {136,0}, {137,0}, {138,0}, {139,0}, {140,0}, {141,0}, {142,0}, {143,0}, {144,0}, {145,0}, {146,0}, {147,0}, {148,0}, {149,0}, {150,0}, {151,0}, {152,0}, {153,0}, {154,0}, {155,0}, {156,0}, {157,0}, {158,0}, {159,0}, {160,0}, {161,0}, {162,0}, {163,0}, {164,0}, {165,0}, {166,0}, {167,0}, {168,0}, {169,0}, {170,0}, {171,0}, {172,0}, {173,0}, {174,0}, {175,0}, {176,0}, {177,0}, {178,0}, {179,0}, {180,0}, {181,0}, {182,0}, {183,0}, {184,0}, {185,0}, {186,0}, {187,0}, {188,0}, {189,0}, {190,0}, {191,0}, {192,0}, {193,0}, {194,0}, {195,0}, {196,0}, {197,0}, {198,0}, {199,0}, {200,0}, {201,0}, {202,0}, {203,0}, {204,0}, {205,0}, {206,0}, {207,0}, {208,0}, {209,0}, {210,0}, {211,0}, {212,0}, {213,0}, {214,0}, {215,0}, {216,0}, {217,0}, {218,0}, {219,0}, {220,0}, {221,0}, {222,0}, {223,0}, {224,0}, {225,0}, {226,0}, {227,0}, {228,0}, {229,0}, {230,0}, {231,0}, {232,0}, {233,0}, {234,0}, {235,0}, {236,0}, {237,0}, {238,0}, {239,0}, {240,0}, {241,0}, {242,0}, {243,0}, {244,0}, {245,0}, {246,0}, {247,0}, {248,0}, {249,0}, {250,0}, {251,0}, {252,0}, {253,0}, {254,0}, {255,0}, {256,0}, {257,0}, {258,0}, {259,0}, {260,0}, {261,0}, {262,0}, {263,0}, {264,0}, {265,0}, {266,0}, {267,0}, {268,0}, {269,0}, {270,0}, {271,0}, {272,0}, {273,0}, {274,0}, {275,0}, {276,0}, {277,0}, {278,0}, {279,0}, {280,0}, {281,0}, {282,0}, {283,0}, {284,0}, {285,0}, {286,0}, {287,0}, {288,0}, {289,0}, {290,0}, {291,0}, {292,0}, {293,0}, {294,0}, {295,0}, {296,0}, {297,0}, {298,0}, {299,0}, {300,0}, {301,0}, {302,0}, {303,0}, {304,0}, {305,0}, {306,0}, {307,0}, {308,0}, {309,0}, {310,0}, {311,0}, {312,0}, {313,0}, {314,0}, {315,0}, {316,0}, {317,0}, {318,0}, {319,0}, {320,0}, {321,0}, {322,0}, {323,0}, {324,0}, {325,0}, {326,0}, {327,0}, {328,0}, {329,0}, {330,0}, {331,0}, {332,0}, {333,0}, {334,0}, {335,0}, {336,0}, {337,0}, {338,0}, {339,0}, {340,0}, {341,0}, {342,0}, {343,0}, {344,0}, {345,0}, {346,0}, {347,0}, {348,0}, {349,0}, {350,0}, {351,0}, {352,0}, {353,0}, {354,0}, {355,0}, {356,0}, {357,0}, {358,0}, {359,0}, {360,0}, {361,0}, {362,0}, {363,0}, {364,0}, {365,0}, {366,0}, {367,0}, {368,0}, {369,0}, {370,0}, {371,0}, {372,0}, {373,0}, {374,0}, {375,0}, {376,0}, {377,0}, {378,0}, {379,0}, {380,0}, {381,0}, {382,0}, {383,0}, {384,0}, {385,0}, {386,0}, {387,0}, {388,0}, {389,0}, {390,0}, {391,0}, {392,0}, {393,0}, {394,0}, {395,0}, {396,0}, {397,0}, {398,0}, {399,0}, {400,0}, {401,0}, {402,0}, {403,0}, {404,0}, {405,0}, {406,0}, {407,0}, {408,0}, {409,0}, {410,0}, {411,0}, {412,0}, {413,0}, {414,0}, {415,0}, {416,0}, {417,0}, {418,0}, {419,0}, {420,0}, {421,0}, {422,0}, {423,0}, {424,0}, {425,0}, {426,0}, {427,0}, {428,0}, {429,0}, {430,0}, {431,0}, {432,0}, {433,0}, {434,0}, {435,0}, {436,0}, {437,0}, {438,0}, {439,0}, {440,0}, {441,0}, {442,0}, {443,0}, {444,0}, {445,0}, {446,0}, {447,0}, {448,0}, {449,0}, {450,0}, {451,0}, {452,0}, {453,0}, {454,0}, {455,0}, {456,0}, {457,0}, {458,0}, {459,0}, {460,0}, {461,0}, {462,0}, {463,0}, {464,0}, {465,0}, {466,0}, {467,0}, {468,0}, {469,0}, {470,0}, {471,0}, {472,0}, {473,0}, {474,0}, {475,0}, {476,0}, {477,0}, {478,0}, {479,0}, {480,0}, {481,0}, {482,0}, {483,0}, {484,0}, {485,0}, {486,0}, {487,0}, {488,0}, {489,0}, {490,0}, {491,0}, {492,0}, {493,0}, {494,0}, {495,0}, {496,0}, {497,0}, {498,0}, {499,0}, {500,0}, {501,0}, {502,0}, {503,0}, {504,0}, {505,0}, {506,0}, {507,0}, {508,0}, {509,0}, {510,0}, {511,0}]}, {name,"replication_ns_1@10.88.23.62"}, {takeover,false}] INFO REPORT <5901.939.0> 2011-07-13 23:54:30 =============================================================================== ns_1@10.88.165.188:<5901.939.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.88.165.188', [{last_heard,{1310,615665,482193}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,27343208}, {processes,8145488}, {processes_used,8136872}, {system,19197720}, {atom,823481}, {atom_used,817012}, {binary,456440}, {code,7791057}, {ets,823768}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,289}, {memory_data,{644874240,636018688,{<5901.943.0>,372072}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 4180 kB\nBuffers: 18600 kB\nCached: 369804 kB\nSwapCached: 0 kB\nActive: 276012 kB\nInactive: 272652 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 4180 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 348 kB\nWriteback: 0 kB\nAnonPages: 160296 kB\nMapped: 13688 kB\nSlab: 36440 kB\nPageTables: 4064 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 648124 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,378679296}, {buffered_memory,19046400}, {free_memory,4280320}, {total_memory,644874240}]}, {statistics, [{wall_clock,{279023,3}}, {context_switches,{45756,0}}, {garbage_collection,{9375,17396333,0}}, {io,{{input,7998141},{output,2861446}}}, {reductions,{5794581,1384946}}, {run_queue,0}, {runtime,{490,170}}]}]}, {'ns_1@10.88.23.62', [{last_heard,{1310,615669,478443}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,31798984}, {processes,11723048}, {processes_used,11713848}, {system,20075936}, {atom,899193}, {atom_used,873961}, {binary,347160}, {code,8557947}, {ets,903112}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {inets,"5.5.2"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,392}, {memory_data,{644874240,637906944,{<5990.234.0>,372072}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 4304 kB\nBuffers: 17912 kB\nCached: 371096 kB\nSwapCached: 0 kB\nActive: 276596 kB\nInactive: 271928 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 4304 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 776 kB\nWriteback: 0 kB\nAnonPages: 159556 kB\nMapped: 13692 kB\nSlab: 36536 kB\nPageTables: 4072 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 650600 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,380002304}, {buffered_memory,18341888}, {free_memory,4407296}, {total_memory,644874240}]}, {statistics, [{wall_clock,{377501,2}}, {context_switches,{66368,0}}, {garbage_collection,{13063,30570665,0}}, {io,{{input,9783571},{output,4808737}}}, {reductions,{10137861,3533615}}, {run_queue,0}, {runtime,{710,270}}]}]}] INFO REPORT <5901.919.0> 2011-07-13 23:54:44 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.919.0> 2011-07-13 23:55:17 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.939.0> 2011-07-13 23:55:30 =============================================================================== ns_1@10.88.165.188:<5901.939.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.88.165.188', [{last_heard,{1310,615725,481956}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,28220328}, {processes,8812432}, {processes_used,8803816}, {system,19407896}, {atom,823481}, {atom_used,817012}, {binary,304472}, {code,7791057}, {ets,1184000}]}, {system_stats, [{cpu_utilization_rate,0.9803921568627451}, {swap_total,0}, {swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,349}, {memory_data,{644874240,639922176,{<5901.943.0>,1343216}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 4616 kB\nBuffers: 18692 kB\nCached: 368528 kB\nSwapCached: 0 kB\nActive: 276948 kB\nInactive: 271168 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 4616 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 212 kB\nWriteback: 0 kB\nAnonPages: 160932 kB\nMapped: 13696 kB\nSlab: 36384 kB\nPageTables: 4064 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 647536 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,377372672}, {buffered_memory,19140608}, {free_memory,4726784}, {total_memory,644874240}]}, {statistics, [{wall_clock,{339027,6}}, {context_switches,{57641,0}}, {garbage_collection,{11926,26738803,0}}, {io,{{input,8183486},{output,3808806}}}, {reductions,{9278496,842179}}, {run_queue,0}, {runtime,{590,30}}]}]}, {'ns_1@10.88.23.62', [{last_heard,{1310,615729,465677}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,30891080}, {processes,10399888}, {processes_used,10386392}, {system,20491192}, {atom,899193}, {atom_used,873961}, {binary,429224}, {code,8557947}, {ets,1237304}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {inets,"5.5.2"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,452}, {memory_data,{644874240,640086016,{<5990.234.0>,1343216}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 5132 kB\nBuffers: 18040 kB\nCached: 368796 kB\nSwapCached: 0 kB\nActive: 278332 kB\nInactive: 269220 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 5132 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 92 kB\nWriteback: 0 kB\nAnonPages: 160756 kB\nMapped: 13700 kB\nSlab: 36448 kB\nPageTables: 4080 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 653768 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,377647104}, {buffered_memory,18472960}, {free_memory,5255168}, {total_memory,644874240}]}, {statistics, [{wall_clock,{437501,1}}, {context_switches,{86313,0}}, {garbage_collection,{16968,48099489,0}}, {io,{{input,10144595},{output,5713453}}}, {reductions,{27390790,3025621}}, {run_queue,0}, {runtime,{1250,130}}]}]}] INFO REPORT <5901.1085.0> 2011-07-13 23:55:48 =============================================================================== ns_1@10.88.165.188:<5901.1085.0>:stats_collector:83: Stats for bucket "default": accepting_conns 1 auth_cmds 2 auth_errors 0 bucket_active_conns 1 bucket_conns 4 bytes_read 79173 bytes_written 1773470 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 16 connection_structures 13 curr_connections 13 curr_items 0 curr_items_tot 0 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_bg_fetched 0 ep_commit_num 0 ep_commit_time 0 ep_commit_time_total 0 ep_data_age 0 ep_data_age_highwat 0 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 0 ep_dbname /opt/membase/var/lib/membase/data/default-data/default ep_dbshards 4 ep_diskqueue_drain 0 ep_diskqueue_fill 0 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 0 ep_flush_duration_total 0 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 0 ep_io_num_write 0 ep_io_read_bytes 0 ep_io_write_bytes 0 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 0 ep_kv_size 8192 ep_latency_arith_cmd 0 ep_latency_get_cmd 0 ep_latency_store_cmd 0 ep_max_data_size 515899392 ep_max_txn_size 1000 ep_mem_high_wat 386924544 ep_mem_low_wat 309539635 ep_min_data_age 0 ep_num_active_non_resident 0 ep_num_checkpoint_remover_runs20 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 26919552 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 0 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 0 ep_total_del_items 0 ep_total_enqueued 0 ep_total_new_items 0 ep_total_persisted 0 ep_uncommitted_items 0 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.0_33_ged2a54a ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 45712 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 26927744 pid 2654 pointer_size 64 rejected_conns 0 rusage_system 0.124007 rusage_user 0.604037 tap_checkpoint_start_received 512 tap_checkpoint_start_sent 512 tap_connect_received 1 tap_opaque_received 2 tap_opaque_sent 2 threads 4 time 1310615747 total_connections 13 uptime 139 vb_active_curr_items 0 vb_active_eject 0 vb_active_ht_memory 12836864 vb_active_itm_memory 0 vb_active_num 512 vb_active_num_non_resident 0 vb_active_ops_create 0 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 0 vb_active_queue_age 0 vb_active_queue_drain 0 vb_active_queue_fill 0 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 12836864 vb_replica_itm_memory 0 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_461_gf99c147 INFO REPORT <5901.919.0> 2011-07-13 23:55:55 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.939.0> 2011-07-13 23:56:30 =============================================================================== ns_1@10.88.165.188:<5901.939.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.88.165.188', [{last_heard,{1310,615785,481835}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,28820264}, {processes,9250432}, {processes_used,9241816}, {system,19569832}, {atom,823481}, {atom_used,817012}, {binary,314664}, {code,7791057}, {ets,1334000}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,409}, {memory_data,{644874240,634376192,{<5901.943.0>,1113960}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 9284 kB\nBuffers: 18792 kB\nCached: 369284 kB\nSwapCached: 0 kB\nActive: 271836 kB\nInactive: 271952 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 9284 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 396 kB\nWriteback: 0 kB\nAnonPages: 155712 kB\nMapped: 13208 kB\nSlab: 36296 kB\nPageTables: 3724 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 641840 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,378146816}, {buffered_memory,19243008}, {free_memory,9506816}, {total_memory,644874240}]}, {statistics, [{wall_clock,{399023,3}}, {context_switches,{69547,0}}, {garbage_collection,{14517,36148395,0}}, {io,{{input,8411577},{output,4734858}}}, {reductions,{12821160,858230}}, {run_queue,0}, {runtime,{730,10}}]}]}, {'ns_1@10.88.23.62', [{last_heard,{1310,615789,478082}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,31986136}, {processes,11457216}, {processes_used,11441976}, {system,20528920}, {atom,899193}, {atom_used,873961}, {binary,311024}, {code,8557947}, {ets,1389864}]}, {system_stats, [{cpu_utilization_rate,1.9801980198019802}, {swap_total,0}, {swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {inets,"5.5.2"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,512}, {memory_data,{644874240,633704448,{<5990.942.0>,688936}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 10172 kB\nBuffers: 18148 kB\nCached: 369572 kB\nSwapCached: 0 kB\nActive: 273372 kB\nInactive: 269996 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 10172 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 372 kB\nWriteback: 0 kB\nAnonPages: 155712 kB\nMapped: 13212 kB\nSlab: 36356 kB\nPageTables: 3744 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 659492 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,378441728}, {buffered_memory,18583552}, {free_memory,10416128}, {total_memory,644874240}]}, {statistics, [{wall_clock,{497502,2}}, {context_switches,{105746,0}}, {garbage_collection,{20454,64431585,0}}, {io,{{input,10434831},{output,6650630}}}, {reductions,{44418360,3058768}}, {run_queue,0}, {runtime,{1750,90}}]}]}] INFO REPORT <5901.919.0> 2011-07-13 23:56:36 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.919.0> 2011-07-13 23:57:14 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.1085.0> 2011-07-13 23:57:28 =============================================================================== ns_1@10.88.165.188:<5901.1085.0>:stats_collector:83: Stats for bucket "default": accepting_conns 1 auth_cmds 2 auth_errors 0 bucket_active_conns 1 bucket_conns 4 bytes_read 86253 bytes_written 3126410 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 16 connection_structures 13 curr_connections 13 curr_items 0 curr_items_tot 0 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_bg_fetched 0 ep_commit_num 0 ep_commit_time 0 ep_commit_time_total 0 ep_data_age 0 ep_data_age_highwat 0 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 0 ep_dbname /opt/membase/var/lib/membase/data/default-data/default ep_dbshards 4 ep_diskqueue_drain 0 ep_diskqueue_fill 0 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 0 ep_flush_duration_total 0 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 0 ep_io_num_write 0 ep_io_read_bytes 0 ep_io_write_bytes 0 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 0 ep_kv_size 8192 ep_latency_arith_cmd 0 ep_latency_get_cmd 0 ep_latency_store_cmd 0 ep_max_data_size 515899392 ep_max_txn_size 1000 ep_mem_high_wat 386924544 ep_mem_low_wat 309539635 ep_min_data_age 0 ep_num_active_non_resident 0 ep_num_checkpoint_remover_runs40 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 26919552 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 0 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 0 ep_total_del_items 0 ep_total_enqueued 0 ep_total_new_items 0 ep_total_persisted 0 ep_uncommitted_items 0 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.0_33_ged2a54a ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 45712 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 26927744 pid 2654 pointer_size 64 rejected_conns 0 rusage_system 0.132008 rusage_user 0.624039 tap_checkpoint_start_received 512 tap_checkpoint_start_sent 512 tap_connect_received 1 tap_opaque_received 2 tap_opaque_sent 2 threads 4 time 1310615848 total_connections 13 uptime 240 vb_active_curr_items 0 vb_active_eject 0 vb_active_ht_memory 12836864 vb_active_itm_memory 0 vb_active_num 512 vb_active_num_non_resident 0 vb_active_ops_create 0 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 0 vb_active_queue_age 0 vb_active_queue_drain 0 vb_active_queue_fill 0 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 12836864 vb_replica_itm_memory 0 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_461_gf99c147 INFO REPORT <5901.939.0> 2011-07-13 23:57:30 =============================================================================== ns_1@10.88.165.188:<5901.939.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.88.165.188', [{last_heard,{1310,615845,481567}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,29304104}, {processes,9545976}, {processes_used,9520400}, {system,19758128}, {atom,823481}, {atom_used,817012}, {binary,353416}, {code,7791057}, {ets,1479864}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,469}, {memory_data,{644874240,636637184,{<5901.943.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 7548 kB\nBuffers: 18988 kB\nCached: 370024 kB\nSwapCached: 0 kB\nActive: 272872 kB\nInactive: 272772 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 7548 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 224 kB\nWriteback: 0 kB\nAnonPages: 156672 kB\nMapped: 13208 kB\nSlab: 36380 kB\nPageTables: 3728 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 642864 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,378904576}, {buffered_memory,19443712}, {free_memory,7729152}, {total_memory,644874240}]}, {statistics, [{wall_clock,{459023,3}}, {context_switches,{82111,0}}, {garbage_collection,{17308,45885892,0}}, {io,{{input,10598032},{output,7500402}}}, {reductions,{16368132,840186}}, {run_queue,0}, {runtime,{940,50}}]}]}, {'ns_1@10.88.23.62', [{last_heard,{1310,615849,475997}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,32171720}, {processes,11388448}, {processes_used,11373208}, {system,20783272}, {atom,899193}, {atom_used,873961}, {binary,420112}, {code,8557947}, {ets,1533152}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {inets,"5.5.2"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,572}, {memory_data,{644874240,635473920,{<5990.234.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 8808 kB\nBuffers: 18348 kB\nCached: 370324 kB\nSwapCached: 0 kB\nActive: 273316 kB\nInactive: 270908 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 8808 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 240 kB\nWriteback: 0 kB\nAnonPages: 155604 kB\nMapped: 13212 kB\nSlab: 36436 kB\nPageTables: 3744 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 648660 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,379211776}, {buffered_memory,18788352}, {free_memory,9019392}, {total_memory,644874240}]}, {statistics, [{wall_clock,{557501,2}}, {context_switches,{125836,0}}, {garbage_collection,{24175,81057021,0}}, {io,{{input,12710964},{output,9304491}}}, {reductions,{61452985,3051736}}, {run_queue,0}, {runtime,{2290,90}}]}]}] INFO REPORT <5901.919.0> 2011-07-13 23:57:31 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.919.0> 2011-07-13 23:57:44 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.919.0> 2011-07-13 23:58:01 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.919.0> 2011-07-13 23:58:29 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.939.0> 2011-07-13 23:58:30 =============================================================================== ns_1@10.88.165.188:<5901.939.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.88.165.188', [{last_heard,{1310,615905,481553}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,29448424}, {processes,9536912}, {processes_used,9511336}, {system,19911512}, {atom,823481}, {atom_used,817012}, {binary,361264}, {code,7791057}, {ets,1623496}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,529}, {memory_data,{644874240,637652992,{<5901.943.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 6680 kB\nBuffers: 19092 kB\nCached: 370776 kB\nSwapCached: 0 kB\nActive: 273132 kB\nInactive: 273592 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 6680 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 64 kB\nWriteback: 0 kB\nAnonPages: 156848 kB\nMapped: 13208 kB\nSlab: 36372 kB\nPageTables: 3728 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 641840 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,379674624}, {buffered_memory,19550208}, {free_memory,6840320}, {total_memory,644874240}]}, {statistics, [{wall_clock,{519026,5}}, {context_switches,{94007,0}}, {garbage_collection,{19959,55265422,0}}, {io,{{input,10877508},{output,8372170}}}, {reductions,{19874640,824539}}, {run_queue,0}, {runtime,{1060,30}}]}]}, {'ns_1@10.88.23.62', [{last_heard,{1310,615909,475450}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,32261568}, {processes,11433984}, {processes_used,11418744}, {system,20827584}, {atom,899193}, {atom_used,873961}, {binary,319104}, {code,8557947}, {ets,1676424}]}, {system_stats, [{cpu_utilization_rate,1.0101010101010102}, {swap_total,0}, {swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {inets,"5.5.2"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,632}, {memory_data,{644874240,636743680,{<5990.234.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 7940 kB\nBuffers: 18456 kB\nCached: 371104 kB\nSwapCached: 0 kB\nActive: 273564 kB\nInactive: 271716 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 7940 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 120 kB\nWriteback: 0 kB\nAnonPages: 155744 kB\nMapped: 13212 kB\nSlab: 36428 kB\nPageTables: 3744 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 648660 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,380010496}, {buffered_memory,18898944}, {free_memory,8130560}, {total_memory,644874240}]}, {statistics, [{wall_clock,{617505,4}}, {context_switches,{145225,0}}, {garbage_collection,{27681,97292186,0}}, {io,{{input,12955155},{output,10340093}}}, {reductions,{78435169,5202096}}, {run_queue,0}, {runtime,{2820,160}}]}]}] INFO REPORT <5901.919.0> 2011-07-13 23:59:05 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.1085.0> 2011-07-13 23:59:08 =============================================================================== ns_1@10.88.165.188:<5901.1085.0>:stats_collector:83: Stats for bucket "default": accepting_conns 1 auth_cmds 2 auth_errors 0 bucket_active_conns 1 bucket_conns 4 bytes_read 93333 bytes_written 4479350 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 16 connection_structures 13 curr_connections 13 curr_items 0 curr_items_tot 0 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_bg_fetched 0 ep_commit_num 0 ep_commit_time 0 ep_commit_time_total 0 ep_data_age 0 ep_data_age_highwat 0 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 0 ep_dbname /opt/membase/var/lib/membase/data/default-data/default ep_dbshards 4 ep_diskqueue_drain 0 ep_diskqueue_fill 0 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 0 ep_flush_duration_total 0 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 0 ep_io_num_write 0 ep_io_read_bytes 0 ep_io_write_bytes 0 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 0 ep_kv_size 8192 ep_latency_arith_cmd 0 ep_latency_get_cmd 0 ep_latency_store_cmd 0 ep_max_data_size 515899392 ep_max_txn_size 1000 ep_mem_high_wat 386924544 ep_mem_low_wat 309539635 ep_min_data_age 0 ep_num_active_non_resident 0 ep_num_checkpoint_remover_runs60 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 26919552 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 0 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 0 ep_total_del_items 0 ep_total_enqueued 0 ep_total_new_items 0 ep_total_persisted 0 ep_uncommitted_items 0 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.0_33_ged2a54a ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 45712 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 26927744 pid 2654 pointer_size 64 rejected_conns 0 rusage_system 0.132008 rusage_user 0.640040 tap_checkpoint_start_received 512 tap_checkpoint_start_sent 512 tap_connect_received 1 tap_opaque_received 2 tap_opaque_sent 2 threads 4 time 1310615948 total_connections 13 uptime 340 vb_active_curr_items 0 vb_active_eject 0 vb_active_ht_memory 12836864 vb_active_itm_memory 0 vb_active_num 512 vb_active_num_non_resident 0 vb_active_ops_create 0 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 0 vb_active_queue_age 0 vb_active_queue_drain 0 vb_active_queue_fill 0 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 12836864 vb_replica_itm_memory 0 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_461_gf99c147 INFO REPORT <5901.939.0> 2011-07-13 23:59:30 =============================================================================== ns_1@10.88.165.188:<5901.939.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.88.165.188', [{last_heard,{1310,615965,481597}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,29915064}, {processes,9882320}, {processes_used,9856744}, {system,20032744}, {atom,823481}, {atom_used,817012}, {binary,337128}, {code,7791057}, {ets,1767016}]}, {system_stats, [{cpu_utilization_rate,0.9900990099009901}, {swap_total,0}, {swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,589}, {memory_data,{644874240,638541824,{<5901.943.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 5316 kB\nBuffers: 19192 kB\nCached: 371536 kB\nSwapCached: 0 kB\nActive: 273392 kB\nInactive: 274340 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 5316 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 112 kB\nWriteback: 0 kB\nAnonPages: 157004 kB\nMapped: 13208 kB\nSlab: 36380 kB\nPageTables: 3728 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 641840 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,380452864}, {buffered_memory,19652608}, {free_memory,5443584}, {total_memory,644874240}]}, {statistics, [{wall_clock,{579024,3}}, {context_switches,{105889,0}}, {garbage_collection,{22516,64699575,0}}, {io,{{input,11106263},{output,9288625}}}, {reductions,{23409582,842046}}, {run_queue,0}, {runtime,{1160,20}}]}]}, {'ns_1@10.88.23.62', [{last_heard,{1310,615969,463379}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,31896560}, {processes,10957784}, {processes_used,10940416}, {system,20938776}, {atom,899193}, {atom_used,873961}, {binary,281672}, {code,8557947}, {ets,1822632}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {inets,"5.5.2"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,692}, {memory_data,{644874240,637632512,{<5990.234.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 6576 kB\nBuffers: 18564 kB\nCached: 371876 kB\nSwapCached: 0 kB\nActive: 273808 kB\nInactive: 272484 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 6576 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 380 kB\nWriteback: 0 kB\nAnonPages: 155892 kB\nMapped: 13212 kB\nSlab: 36452 kB\nPageTables: 3744 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 648660 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,380801024}, {buffered_memory,19009536}, {free_memory,6733824}, {total_memory,644874240}]}, {statistics, [{wall_clock,{677503,1}}, {context_switches,{164625,0}}, {garbage_collection,{31202,113605451,0}}, {io,{{input,13247824},{output,11252598}}}, {reductions,{95451843,5212494}}, {run_queue,0}, {runtime,{3290,140}}]}]}] INFO REPORT <5901.919.0> 2011-07-13 23:59:32 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.919.0> 2011-07-13 23:59:41 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.939.0> 2011-07-14 00:00:30 =============================================================================== ns_1@10.88.165.188:<5901.939.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.88.165.188', [{last_heard,{1310,616025,481540}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,30029088}, {processes,9854688}, {processes_used,9829112}, {system,20174400}, {atom,823481}, {atom_used,817012}, {binary,316488}, {code,7791057}, {ets,1926560}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,649}, {memory_data,{644874240,636637184,{<5901.943.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 7672 kB\nBuffers: 19400 kB\nCached: 361904 kB\nSwapCached: 0 kB\nActive: 280668 kB\nInactive: 264032 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 7672 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 660 kB\nWriteback: 0 kB\nAnonPages: 163448 kB\nMapped: 13696 kB\nSlab: 36588 kB\nPageTables: 4084 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 650000 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,370589696}, {buffered_memory,19865600}, {free_memory,7856128}, {total_memory,644874240}]}, {statistics, [{wall_clock,{639023,3}}, {context_switches,{118236,0}}, {garbage_collection,{25212,74567187,0}}, {io,{{input,13772508},{output,11031327}}}, {reductions,{26991810,836087}}, {run_queue,0}, {runtime,{1320,40}}]}]}, {'ns_1@10.88.23.62', [{last_heard,{1310,616029,478707}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,32088400}, {processes,10952896}, {processes_used,10935528}, {system,21135504}, {atom,899193}, {atom_used,873961}, {binary,320704}, {code,8557947}, {ets,1978688}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {inets,"5.5.2"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,752}, {memory_data,{644874240,635219968,{<5990.234.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 8220 kB\nBuffers: 18800 kB\nCached: 363236 kB\nSwapCached: 0 kB\nActive: 281356 kB\nInactive: 262788 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 8220 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 764 kB\nWriteback: 0 kB\nAnonPages: 162168 kB\nMapped: 13700 kB\nSlab: 36672 kB\nPageTables: 4092 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 656232 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,371953664}, {buffered_memory,19251200}, {free_memory,8417280}, {total_memory,644874240}]}, {statistics, [{wall_clock,{737503,2}}, {context_switches,{184053,0}}, {garbage_collection,{34752,128987158,0}}, {io,{{input,15978342},{output,13024732}}}, {reductions,{112251767,3036689}}, {run_queue,0}, {runtime,{3800,90}}]}]}] INFO REPORT <5901.919.0> 2011-07-14 00:00:40 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.1085.0> 2011-07-14 00:00:48 =============================================================================== ns_1@10.88.165.188:<5901.1085.0>:stats_collector:83: Stats for bucket "default": accepting_conns 1 auth_cmds 2 auth_errors 0 bucket_active_conns 1 bucket_conns 4 bytes_read 100413 bytes_written 5832296 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 16 connection_structures 13 curr_connections 13 curr_items 0 curr_items_tot 0 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_bg_fetched 0 ep_commit_num 0 ep_commit_time 0 ep_commit_time_total 0 ep_data_age 0 ep_data_age_highwat 0 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 0 ep_dbname /opt/membase/var/lib/membase/data/default-data/default ep_dbshards 4 ep_diskqueue_drain 0 ep_diskqueue_fill 0 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 0 ep_flush_duration_total 0 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 0 ep_io_num_write 0 ep_io_read_bytes 0 ep_io_write_bytes 0 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 0 ep_kv_size 8192 ep_latency_arith_cmd 0 ep_latency_get_cmd 0 ep_latency_store_cmd 0 ep_max_data_size 515899392 ep_max_txn_size 1000 ep_mem_high_wat 386924544 ep_mem_low_wat 309539635 ep_min_data_age 0 ep_num_active_non_resident 0 ep_num_checkpoint_remover_runs80 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 26919552 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 0 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 0 ep_total_del_items 0 ep_total_enqueued 0 ep_total_new_items 0 ep_total_persisted 0 ep_uncommitted_items 0 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.0_33_ged2a54a ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 45712 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 26927744 pid 2654 pointer_size 64 rejected_conns 0 rusage_system 0.140008 rusage_user 0.664041 tap_checkpoint_start_received 512 tap_checkpoint_start_sent 512 tap_connect_received 1 tap_opaque_received 2 tap_opaque_sent 2 threads 4 time 1310616048 total_connections 13 uptime 440 vb_active_curr_items 0 vb_active_eject 0 vb_active_ht_memory 12836864 vb_active_itm_memory 0 vb_active_num 512 vb_active_num_non_resident 0 vb_active_ops_create 0 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 0 vb_active_queue_age 0 vb_active_queue_drain 0 vb_active_queue_fill 0 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 12836864 vb_replica_itm_memory 0 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_461_gf99c147 INFO REPORT <5901.919.0> 2011-07-14 00:00:55 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.919.0> 2011-07-14 00:01:02 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.939.0> 2011-07-14 00:01:30 =============================================================================== ns_1@10.88.165.188:<5901.939.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.88.165.188', [{last_heard,{1310,616085,481526}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,30255864}, {processes,9877416}, {processes_used,9851840}, {system,20378448}, {atom,823481}, {atom_used,817012}, {binary,375296}, {code,7791057}, {ets,2069720}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,709}, {memory_data,{644874240,637272064,{<5901.943.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 6548 kB\nBuffers: 19508 kB\nCached: 362728 kB\nSwapCached: 0 kB\nActive: 281180 kB\nInactive: 264648 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 6548 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 56 kB\nWriteback: 0 kB\nAnonPages: 163592 kB\nMapped: 13696 kB\nSlab: 36564 kB\nPageTables: 4080 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 650000 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,371433472}, {buffered_memory,19976192}, {free_memory,6705152}, {total_memory,644874240}]}, {statistics, [{wall_clock,{699023,3}}, {context_switches,{129876,0}}, {garbage_collection,{27672,83913741,0}}, {io,{{input,14051177},{output,11995060}}}, {reductions,{30473664,831026}}, {run_queue,0}, {runtime,{1430,20}}]}]}, {'ns_1@10.88.23.62', [{last_heard,{1310,616089,463766}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,32230792}, {processes,10909688}, {processes_used,10892320}, {system,21321104}, {atom,899193}, {atom_used,873961}, {binary,355280}, {code,8557947}, {ets,2127608}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {inets,"5.5.2"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,812}, {memory_data,{644874240,637218816,{<5990.234.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 7104 kB\nBuffers: 18908 kB\nCached: 364080 kB\nSwapCached: 0 kB\nActive: 282432 kB\nInactive: 263408 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 7104 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 228 kB\nWriteback: 0 kB\nAnonPages: 162892 kB\nMapped: 13700 kB\nSlab: 36636 kB\nPageTables: 4088 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 656232 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,372817920}, {buffered_memory,19361792}, {free_memory,7274496}, {total_memory,644874240}]}, {statistics, [{wall_clock,{797504,3}}, {context_switches,{202593,0}}, {garbage_collection,{37988,143554803,0}}, {io,{{input,16313716},{output,13990812}}}, {reductions,{128903479,3029526}}, {run_queue,0}, {runtime,{4320,100}}]}]}] INFO REPORT <5901.919.0> 2011-07-14 00:01:53 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.919.0> 2011-07-14 00:02:16 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.1085.0> 2011-07-14 00:02:28 =============================================================================== ns_1@10.88.165.188:<5901.1085.0>:stats_collector:83: Stats for bucket "default": accepting_conns 1 auth_cmds 2 auth_errors 0 bucket_active_conns 1 bucket_conns 4 bytes_read 107493 bytes_written 7185347 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 16 connection_structures 13 curr_connections 13 curr_items 0 curr_items_tot 0 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_bg_fetched 0 ep_commit_num 0 ep_commit_time 0 ep_commit_time_total 0 ep_data_age 0 ep_data_age_highwat 0 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 0 ep_dbname /opt/membase/var/lib/membase/data/default-data/default ep_dbshards 4 ep_diskqueue_drain 0 ep_diskqueue_fill 0 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 0 ep_flush_duration_total 0 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 0 ep_io_num_write 0 ep_io_read_bytes 0 ep_io_write_bytes 0 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 0 ep_kv_size 8192 ep_latency_arith_cmd 0 ep_latency_get_cmd 0 ep_latency_store_cmd 0 ep_max_data_size 515899392 ep_max_txn_size 1000 ep_mem_high_wat 386924544 ep_mem_low_wat 309539635 ep_min_data_age 0 ep_num_active_non_resident 0 ep_num_checkpoint_remover_runs100 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 26919552 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 0 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 0 ep_total_del_items 0 ep_total_enqueued 0 ep_total_new_items 0 ep_total_persisted 0 ep_uncommitted_items 0 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.0_33_ged2a54a ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 45712 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 26927744 pid 2654 pointer_size 64 rejected_conns 0 rusage_system 0.156009 rusage_user 0.676042 tap_checkpoint_start_received 512 tap_checkpoint_start_sent 512 tap_connect_received 1 tap_opaque_received 2 tap_opaque_sent 2 threads 4 time 1310616148 total_connections 13 uptime 540 vb_active_curr_items 0 vb_active_eject 0 vb_active_ht_memory 12836864 vb_active_itm_memory 0 vb_active_num 512 vb_active_num_non_resident 0 vb_active_ops_create 0 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 0 vb_active_queue_age 0 vb_active_queue_drain 0 vb_active_queue_fill 0 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 12836864 vb_replica_itm_memory 0 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_461_gf99c147 INFO REPORT <5901.939.0> 2011-07-14 00:02:30 =============================================================================== ns_1@10.88.165.188:<5901.939.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.88.165.188', [{last_heard,{1310,616145,481551}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,30363224}, {processes,9861608}, {processes_used,9836032}, {system,20501616}, {atom,823481}, {atom_used,817012}, {binary,352624}, {code,7791057}, {ets,2213880}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,769}, {memory_data,{644874240,638169088,{<5901.943.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 5820 kB\nBuffers: 19608 kB\nCached: 363488 kB\nSwapCached: 0 kB\nActive: 281404 kB\nInactive: 265384 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 5820 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 396 kB\nWriteback: 0 kB\nAnonPages: 163740 kB\nMapped: 13696 kB\nSlab: 36532 kB\nPageTables: 4084 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 650000 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,372211712}, {buffered_memory,20078592}, {free_memory,5959680}, {total_memory,644874240}]}, {statistics, [{wall_clock,{759026,3}}, {context_switches,{141471,0}}, {garbage_collection,{30022,93103910,0}}, {io,{{input,14215310},{output,12903120}}}, {reductions,{33922966,838477}}, {run_queue,0}, {runtime,{1540,30}}]}]}, {'ns_1@10.88.23.62', [{last_heard,{1310,616149,474131}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,32414496}, {processes,10944792}, {processes_used,10927424}, {system,21469704}, {atom,899193}, {atom_used,873961}, {binary,358992}, {code,8557947}, {ets,2268704}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {inets,"5.5.2"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,872}, {memory_data,{644874240,638107648,{<5990.234.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 6244 kB\nBuffers: 19012 kB\nCached: 364852 kB\nSwapCached: 0 kB\nActive: 282088 kB\nInactive: 264196 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 6244 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 392 kB\nWriteback: 0 kB\nAnonPages: 162460 kB\nMapped: 13700 kB\nSlab: 36624 kB\nPageTables: 4088 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 656232 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,373608448}, {buffered_memory,19468288}, {free_memory,6393856}, {total_memory,644874240}]}, {statistics, [{wall_clock,{857503,2}}, {context_switches,{221103,0}}, {garbage_collection,{41261,158012846,0}}, {io,{{input,16587164},{output,14837252}}}, {reductions,{145527426,3030168}}, {run_queue,0}, {runtime,{4820,90}}]}]}] INFO REPORT <5901.919.0> 2011-07-14 00:03:06 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.919.0> 2011-07-14 00:03:12 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.919.0> 2011-07-14 00:03:20 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.939.0> 2011-07-14 00:03:30 =============================================================================== ns_1@10.88.165.188:<5901.939.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.88.165.188', [{last_heard,{1310,616205,481783}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,30273968}, {processes,9589152}, {processes_used,9563576}, {system,20684816}, {atom,823481}, {atom_used,817012}, {binary,386688}, {code,7791057}, {ets,2359376}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,829}, {memory_data,{644874240,637657088,{<5901.943.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 6676 kB\nBuffers: 19812 kB\nCached: 362172 kB\nSwapCached: 0 kB\nActive: 281468 kB\nInactive: 264352 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 6676 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 176 kB\nWriteback: 0 kB\nAnonPages: 163892 kB\nMapped: 13696 kB\nSlab: 36628 kB\nPageTables: 4084 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 660244 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,370864128}, {buffered_memory,20287488}, {free_memory,6836224}, {total_memory,644874240}]}, {statistics, [{wall_clock,{819027,5}}, {context_switches,{153932,0}}, {garbage_collection,{32591,102878621,0}}, {io,{{input,16928609},{output,16046868}}}, {reductions,{37482255,825988}}, {run_queue,0}, {runtime,{1680,30}}]}]}, {'ns_1@10.88.23.62', [{last_heard,{1310,616209,477821}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,32453696}, {processes,10926176}, {processes_used,10908808}, {system,21527520}, {atom,899193}, {atom_used,873961}, {binary,366224}, {code,8557947}, {ets,2317560}]}, {system_stats,[{cpu_utilization_rate,1.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {inets,"5.5.2"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,932}, {memory_data,{644874240,638513152,{<5990.234.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 6212 kB\nBuffers: 19224 kB\nCached: 364576 kB\nSwapCached: 0 kB\nActive: 282324 kB\nInactive: 264076 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 6212 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 232 kB\nWriteback: 0 kB\nAnonPages: 162612 kB\nMapped: 13700 kB\nSlab: 36676 kB\nPageTables: 4088 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 656820 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,373325824}, {buffered_memory,19685376}, {free_memory,6361088}, {total_memory,644874240}]}, {statistics, [{wall_clock,{917503,2}}, {context_switches,{240379,0}}, {garbage_collection,{44675,172963349,0}}, {io,{{input,19261870},{output,18076887}}}, {reductions,{162231388,3001687}}, {run_queue,0}, {runtime,{5360,80}}]}]}] INFO REPORT <5901.919.0> 2011-07-14 00:03:56 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.1085.0> 2011-07-14 00:04:08 =============================================================================== ns_1@10.88.165.188:<5901.1085.0>:stats_collector:83: Stats for bucket "default": accepting_conns 1 auth_cmds 2 auth_errors 0 bucket_active_conns 1 bucket_conns 4 bytes_read 114573 bytes_written 8538507 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 16 connection_structures 13 curr_connections 13 curr_items 0 curr_items_tot 0 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_bg_fetched 0 ep_commit_num 0 ep_commit_time 0 ep_commit_time_total 0 ep_data_age 0 ep_data_age_highwat 0 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 0 ep_dbname /opt/membase/var/lib/membase/data/default-data/default ep_dbshards 4 ep_diskqueue_drain 0 ep_diskqueue_fill 0 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 0 ep_flush_duration_total 0 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 0 ep_io_num_write 0 ep_io_read_bytes 0 ep_io_write_bytes 0 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 0 ep_kv_size 8192 ep_latency_arith_cmd 0 ep_latency_get_cmd 0 ep_latency_store_cmd 0 ep_max_data_size 515899392 ep_max_txn_size 1000 ep_mem_high_wat 386924544 ep_mem_low_wat 309539635 ep_min_data_age 0 ep_num_active_non_resident 0 ep_num_checkpoint_remover_runs120 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 26919552 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 0 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 0 ep_total_del_items 0 ep_total_enqueued 0 ep_total_new_items 0 ep_total_persisted 0 ep_uncommitted_items 0 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.0_33_ged2a54a ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 45712 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 26927744 pid 2654 pointer_size 64 rejected_conns 0 rusage_system 0.156009 rusage_user 0.704044 tap_checkpoint_start_received 512 tap_checkpoint_start_sent 512 tap_connect_received 1 tap_opaque_received 2 tap_opaque_sent 2 threads 4 time 1310616248 total_connections 13 uptime 640 vb_active_curr_items 0 vb_active_eject 0 vb_active_ht_memory 12836864 vb_active_itm_memory 0 vb_active_num 512 vb_active_num_non_resident 0 vb_active_ops_create 0 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 0 vb_active_queue_age 0 vb_active_queue_drain 0 vb_active_queue_fill 0 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 12836864 vb_replica_itm_memory 0 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_461_gf99c147 INFO REPORT <5901.919.0> 2011-07-14 00:04:22 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.939.0> 2011-07-14 00:04:30 =============================================================================== ns_1@10.88.165.188:<5901.939.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.88.165.188', [{last_heard,{1310,616265,484151}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,30256392}, {processes,9519256}, {processes_used,9493680}, {system,20737136}, {atom,823481}, {atom_used,817012}, {binary,386736}, {code,7791057}, {ets,2409504}]}, {system_stats, [{cpu_utilization_rate,1.0101010101010102}, {swap_total,0}, {swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,889}, {memory_data,{644874240,638410752,{<5901.943.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 5436 kB\nBuffers: 19916 kB\nCached: 362936 kB\nSwapCached: 0 kB\nActive: 281664 kB\nInactive: 265148 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 5436 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 108 kB\nWriteback: 0 kB\nAnonPages: 164000 kB\nMapped: 13696 kB\nSlab: 36600 kB\nPageTables: 4084 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 650000 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,371646464}, {buffered_memory,20393984}, {free_memory,5566464}, {total_memory,644874240}]}, {statistics, [{wall_clock,{879025,3}}, {context_switches,{165582,0}}, {garbage_collection,{35073,112212528,0}}, {io,{{input,17147374},{output,16963698}}}, {reductions,{40989714,860351}}, {run_queue,0}, {runtime,{1790,50}}]}]}, {'ns_1@10.88.23.62', [{last_heard,{1310,616269,479277}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,32478936}, {processes,10784504}, {processes_used,10767136}, {system,21694432}, {atom,899193}, {atom_used,873961}, {binary,387384}, {code,8557947}, {ets,2461392}]}, {system_stats,[{cpu_utilization_rate,1.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {inets,"5.5.2"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,992}, {memory_data,{644874240,639401984,{<5990.234.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 4972 kB\nBuffers: 19332 kB\nCached: 365360 kB\nSwapCached: 0 kB\nActive: 282520 kB\nInactive: 264872 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 4972 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 128 kB\nWriteback: 0 kB\nAnonPages: 162716 kB\nMapped: 13700 kB\nSlab: 36676 kB\nPageTables: 4088 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 656232 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,374128640}, {buffered_memory,19795968}, {free_memory,5091328}, {total_memory,644874240}]}, {statistics, [{wall_clock,{977502,2}}, {context_switches,{258925,0}}, {garbage_collection,{47897,187501313,0}}, {io,{{input,19537478},{output,18993692}}}, {reductions,{178883015,5214638}}, {run_queue,0}, {runtime,{5860,140}}]}]}] INFO REPORT <5901.919.0> 2011-07-14 00:05:15 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.939.0> 2011-07-14 00:05:30 =============================================================================== ns_1@10.88.165.188:<5901.939.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.88.165.188', [{last_heard,{1310,616325,483528}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,30645816}, {processes,9833048}, {processes_used,9807472}, {system,20812768}, {atom,823481}, {atom_used,817012}, {binary,317160}, {code,7791057}, {ets,2553032}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,949}, {memory_data,{644874240,639688704,{<5901.943.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 4692 kB\nBuffers: 20020 kB\nCached: 363692 kB\nSwapCached: 0 kB\nActive: 281832 kB\nInactive: 265932 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 4692 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 500 kB\nWriteback: 0 kB\nAnonPages: 164092 kB\nMapped: 13696 kB\nSlab: 36596 kB\nPageTables: 4084 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 650000 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,372420608}, {buffered_memory,20500480}, {free_memory,4804608}, {total_memory,644874240}]}, {statistics, [{wall_clock,{939024,3}}, {context_switches,{177162,0}}, {garbage_collection,{37576,121423057,0}}, {io,{{input,17315529},{output,17930609}}}, {reductions,{44450457,829888}}, {run_queue,0}, {runtime,{1920,40}}]}]}, {'ns_1@10.88.23.62', [{last_heard,{1310,616329,477823}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,33525224}, {processes,11534544}, {processes_used,11520224}, {system,21990680}, {atom,899193}, {atom_used,873961}, {binary,534480}, {code,8557947}, {ets,2606680}]}, {system_stats, [{cpu_utilization_rate,0.9900990099009901}, {swap_total,0}, {swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {inets,"5.5.2"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1052}, {memory_data,{644874240,640290816,{<5990.234.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 4992 kB\nBuffers: 19440 kB\nCached: 365104 kB\nSwapCached: 0 kB\nActive: 282564 kB\nInactive: 264760 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 4992 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 508 kB\nWriteback: 0 kB\nAnonPages: 162824 kB\nMapped: 13700 kB\nSlab: 36684 kB\nPageTables: 4088 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 656232 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,373866496}, {buffered_memory,19906560}, {free_memory,5111808}, {total_memory,644874240}]}, {statistics, [{wall_clock,{1037503,3}}, {context_switches,{277427,0}}, {garbage_collection,{51105,201982509,0}}, {io,{{input,19863729},{output,19857111}}}, {reductions,{195493457,3032122}}, {run_queue,0}, {runtime,{6370,130}}]}]}] INFO REPORT <5901.1085.0> 2011-07-14 00:05:48 =============================================================================== ns_1@10.88.165.188:<5901.1085.0>:stats_collector:83: Stats for bucket "default": accepting_conns 1 auth_cmds 2 auth_errors 0 bucket_active_conns 1 bucket_conns 4 bytes_read 121653 bytes_written 9891667 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 16 connection_structures 13 curr_connections 13 curr_items 0 curr_items_tot 0 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_bg_fetched 0 ep_commit_num 0 ep_commit_time 0 ep_commit_time_total 0 ep_data_age 0 ep_data_age_highwat 0 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 0 ep_dbname /opt/membase/var/lib/membase/data/default-data/default ep_dbshards 4 ep_diskqueue_drain 0 ep_diskqueue_fill 0 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 0 ep_flush_duration_total 0 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 0 ep_io_num_write 0 ep_io_read_bytes 0 ep_io_write_bytes 0 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 0 ep_kv_size 8192 ep_latency_arith_cmd 0 ep_latency_get_cmd 0 ep_latency_store_cmd 0 ep_max_data_size 515899392 ep_max_txn_size 1000 ep_mem_high_wat 386924544 ep_mem_low_wat 309539635 ep_min_data_age 0 ep_num_active_non_resident 0 ep_num_checkpoint_remover_runs140 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 26919552 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 0 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 0 ep_total_del_items 0 ep_total_enqueued 0 ep_total_new_items 0 ep_total_persisted 0 ep_uncommitted_items 0 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.0_33_ged2a54a ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 45712 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 26927744 pid 2654 pointer_size 64 rejected_conns 0 rusage_system 0.172010 rusage_user 0.728045 tap_checkpoint_start_received 512 tap_checkpoint_start_sent 512 tap_connect_received 1 tap_opaque_received 2 tap_opaque_sent 2 threads 4 time 1310616347 total_connections 13 uptime 739 vb_active_curr_items 0 vb_active_eject 0 vb_active_ht_memory 12836864 vb_active_itm_memory 0 vb_active_num 512 vb_active_num_non_resident 0 vb_active_ops_create 0 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 0 vb_active_queue_age 0 vb_active_queue_drain 0 vb_active_queue_fill 0 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 12836864 vb_replica_itm_memory 0 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_461_gf99c147 INFO REPORT <5901.919.0> 2011-07-14 00:06:04 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.919.0> 2011-07-14 00:06:25 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.939.0> 2011-07-14 00:06:30 =============================================================================== ns_1@10.88.165.188:<5901.939.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.88.165.188', [{last_heard,{1310,616385,483211}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,30927872}, {processes,9895064}, {processes_used,9869488}, {system,21032808}, {atom,823481}, {atom_used,817012}, {binary,387416}, {code,7791057}, {ets,2699832}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1009}, {memory_data,{644874240,625741824,{<5901.943.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 18204 kB\nBuffers: 20220 kB\nCached: 356520 kB\nSwapCached: 0 kB\nActive: 275032 kB\nInactive: 259700 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 18204 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 200 kB\nWriteback: 0 kB\nAnonPages: 157992 kB\nMapped: 13208 kB\nSlab: 36484 kB\nPageTables: 3728 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 641840 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,365076480}, {buffered_memory,20705280}, {free_memory,18640896}, {total_memory,644874240}]}, {statistics, [{wall_clock,{999024,3}}, {context_switches,{189480,0}}, {garbage_collection,{40214,131466044,0}}, {io,{{input,19983700},{output,20380019}}}, {reductions,{48092975,882788}}, {run_queue,0}, {runtime,{2030,10}}]}]}, {'ns_1@10.88.23.62', [{last_heard,{1310,616389,480266}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,32699016}, {processes,10746184}, {processes_used,10728816}, {system,21952832}, {atom,899193}, {atom_used,873961}, {binary,350360}, {code,8557947}, {ets,2752024}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {inets,"5.5.2"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1112}, {memory_data,{644874240,627576832,{<5990.234.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 16404 kB\nBuffers: 19660 kB\nCached: 359984 kB\nSwapCached: 0 kB\nActive: 276560 kB\nInactive: 260312 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 16404 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 184 kB\nWriteback: 0 kB\nAnonPages: 157280 kB\nMapped: 13212 kB\nSlab: 36584 kB\nPageTables: 3748 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 648072 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,368623616}, {buffered_memory,20131840}, {free_memory,16797696}, {total_memory,644874240}]}, {statistics, [{wall_clock,{1097502,2}}, {context_switches,{296692,0}}, {garbage_collection,{54581,217310361,0}}, {io,{{input,22645992},{output,22269708}}}, {reductions,{212306547,5250827}}, {run_queue,0}, {runtime,{6900,130}}]}]}] INFO REPORT <5901.919.0> 2011-07-14 00:06:52 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.919.0> 2011-07-14 00:07:01 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.1085.0> 2011-07-14 00:07:28 =============================================================================== ns_1@10.88.165.188:<5901.1085.0>:stats_collector:83: Stats for bucket "default": accepting_conns 1 auth_cmds 2 auth_errors 0 bucket_active_conns 1 bucket_conns 4 bytes_read 128733 bytes_written 11244929 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 16 connection_structures 13 curr_connections 13 curr_items 0 curr_items_tot 0 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_bg_fetched 0 ep_commit_num 0 ep_commit_time 0 ep_commit_time_total 0 ep_data_age 0 ep_data_age_highwat 0 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 0 ep_dbname /opt/membase/var/lib/membase/data/default-data/default ep_dbshards 4 ep_diskqueue_drain 0 ep_diskqueue_fill 0 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 0 ep_flush_duration_total 0 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 0 ep_io_num_write 0 ep_io_read_bytes 0 ep_io_write_bytes 0 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 0 ep_kv_size 8192 ep_latency_arith_cmd 0 ep_latency_get_cmd 0 ep_latency_store_cmd 0 ep_max_data_size 515899392 ep_max_txn_size 1000 ep_mem_high_wat 386924544 ep_mem_low_wat 309539635 ep_min_data_age 0 ep_num_active_non_resident 0 ep_num_checkpoint_remover_runs160 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 26919552 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 0 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 0 ep_total_del_items 0 ep_total_enqueued 0 ep_total_new_items 0 ep_total_persisted 0 ep_uncommitted_items 0 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.0_33_ged2a54a ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 45712 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 26927744 pid 2654 pointer_size 64 rejected_conns 0 rusage_system 0.180011 rusage_user 0.760047 tap_checkpoint_start_received 512 tap_checkpoint_start_sent 512 tap_connect_received 1 tap_opaque_received 2 tap_opaque_sent 2 threads 4 time 1310616447 total_connections 13 uptime 839 vb_active_curr_items 0 vb_active_eject 0 vb_active_ht_memory 12836864 vb_active_itm_memory 0 vb_active_num 512 vb_active_num_non_resident 0 vb_active_ops_create 0 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 0 vb_active_queue_age 0 vb_active_queue_drain 0 vb_active_queue_fill 0 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 12836864 vb_replica_itm_memory 0 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_461_gf99c147 INFO REPORT <5901.939.0> 2011-07-14 00:07:30 =============================================================================== ns_1@10.88.165.188:<5901.939.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.88.165.188', [{last_heard,{1310,616445,482961}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,31146616}, {processes,9936400}, {processes_used,9910824}, {system,21210216}, {atom,823481}, {atom_used,817012}, {binary,419528}, {code,7791057}, {ets,2843336}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1069}, {memory_data,{644874240,626233344,{<5901.943.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 17460 kB\nBuffers: 20320 kB\nCached: 357276 kB\nSwapCached: 0 kB\nActive: 275264 kB\nInactive: 260424 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 17460 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 100 kB\nWriteback: 0 kB\nAnonPages: 158132 kB\nMapped: 13208 kB\nSlab: 36448 kB\nPageTables: 3728 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 641840 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,365850624}, {buffered_memory,20807680}, {free_memory,17879040}, {total_memory,644874240}]}, {statistics, [{wall_clock,{1059024,3}}, {context_switches,{201087,0}}, {garbage_collection,{42683,140686889,0}}, {io,{{input,20252586},{output,21229378}}}, {reductions,{51546725,859546}}, {run_queue,0}, {runtime,{2140,20}}]}]}, {'ns_1@10.88.23.62', [{last_heard,{1310,616449,480971}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,32888840}, {processes,10921328}, {processes_used,10903960}, {system,21967512}, {atom,899193}, {atom_used,873961}, {binary,323376}, {code,8557947}, {ets,2791624}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {inets,"5.5.2"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1172}, {memory_data,{644874240,628457472,{<5990.234.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 15660 kB\nBuffers: 19768 kB\nCached: 360748 kB\nSwapCached: 0 kB\nActive: 276236 kB\nInactive: 261092 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 15660 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 124 kB\nWriteback: 0 kB\nAnonPages: 156840 kB\nMapped: 13212 kB\nSlab: 36580 kB\nPageTables: 3744 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 648072 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,369405952}, {buffered_memory,20242432}, {free_memory,16035840}, {total_memory,644874240}]}, {statistics, [{wall_clock,{1157502,2}}, {context_switches,{315168,0}}, {garbage_collection,{57990,231727959,0}}, {io,{{input,22868471},{output,23213489}}}, {reductions,{228926448,3048655}}, {run_queue,0}, {runtime,{7410,90}}]}]}] INFO REPORT <5901.919.0> 2011-07-14 00:07:39 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.919.0> 2011-07-14 00:08:20 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.939.0> 2011-07-14 00:08:30 =============================================================================== ns_1@10.88.165.188:<5901.939.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.88.165.188', [{last_heard,{1310,616505,482764}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,30910072}, {processes,9716904}, {processes_used,9691328}, {system,21193168}, {atom,823481}, {atom_used,817012}, {binary,361520}, {code,7791057}, {ets,2882264}]}, {system_stats,[{cpu_utilization_rate,1.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1129}, {memory_data,{644874240,627503104,{<5901.943.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 16336 kB\nBuffers: 20428 kB\nCached: 358068 kB\nSwapCached: 0 kB\nActive: 275684 kB\nInactive: 261072 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 16336 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 624 kB\nWriteback: 0 kB\nAnonPages: 158240 kB\nMapped: 13208 kB\nSlab: 36452 kB\nPageTables: 3732 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 641840 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,366661632}, {buffered_memory,20918272}, {free_memory,16728064}, {total_memory,644874240}]}, {statistics, [{wall_clock,{1119023,2}}, {context_switches,{212693,0}}, {garbage_collection,{45122,149904787,0}}, {io,{{input,20416782},{output,22158347}}}, {reductions,{55011901,847169}}, {run_queue,0}, {runtime,{2240,20}}]}]}, {'ns_1@10.88.23.62', [{last_heard,{1310,616509,465407}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,33019832}, {processes,10897656}, {processes_used,10880288}, {system,22122176}, {atom,899193}, {atom_used,873961}, {binary,328784}, {code,8557947}, {ets,2937448}]}, {system_stats, [{cpu_utilization_rate,0.9900990099009901}, {swap_total,0}, {swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {inets,"5.5.2"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1232}, {memory_data,{644874240,629727232,{<5990.234.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 14660 kB\nBuffers: 19884 kB\nCached: 361560 kB\nSwapCached: 0 kB\nActive: 276488 kB\nInactive: 261900 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 14660 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 632 kB\nWriteback: 0 kB\nAnonPages: 156948 kB\nMapped: 13212 kB\nSlab: 36548 kB\nPageTables: 3744 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 648072 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,370237440}, {buffered_memory,20361216}, {free_memory,15011840}, {total_memory,644874240}]}, {statistics, [{wall_clock,{1217502,2}}, {context_switches,{333687,0}}, {garbage_collection,{61351,246171687,0}}, {io,{{input,23141946},{output,24077042}}}, {reductions,{245547646,3028761}}, {run_queue,0}, {runtime,{7930,120}}]}]}] INFO REPORT <5901.919.0> 2011-07-14 00:08:40 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.919.0> 2011-07-14 00:08:45 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.1085.0> 2011-07-14 00:09:08 =============================================================================== ns_1@10.88.165.188:<5901.1085.0>:stats_collector:83: Stats for bucket "default": accepting_conns 1 auth_cmds 2 auth_errors 0 bucket_active_conns 1 bucket_conns 4 bytes_read 135813 bytes_written 12598199 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 16 connection_structures 13 curr_connections 13 curr_items 0 curr_items_tot 0 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_bg_fetched 0 ep_commit_num 0 ep_commit_time 0 ep_commit_time_total 0 ep_data_age 0 ep_data_age_highwat 0 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 0 ep_dbname /opt/membase/var/lib/membase/data/default-data/default ep_dbshards 4 ep_diskqueue_drain 0 ep_diskqueue_fill 0 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 0 ep_flush_duration_total 0 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 0 ep_io_num_write 0 ep_io_read_bytes 0 ep_io_write_bytes 0 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 0 ep_kv_size 8192 ep_latency_arith_cmd 0 ep_latency_get_cmd 0 ep_latency_store_cmd 0 ep_max_data_size 515899392 ep_max_txn_size 1000 ep_mem_high_wat 386924544 ep_mem_low_wat 309539635 ep_min_data_age 0 ep_num_active_non_resident 0 ep_num_checkpoint_remover_runs180 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 26919552 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 0 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 0 ep_total_del_items 0 ep_total_enqueued 0 ep_total_new_items 0 ep_total_persisted 0 ep_uncommitted_items 0 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.0_33_ged2a54a ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 45712 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 26927744 pid 2654 pointer_size 64 rejected_conns 0 rusage_system 0.188011 rusage_user 0.788049 tap_checkpoint_start_received 512 tap_checkpoint_start_sent 512 tap_connect_received 1 tap_opaque_received 2 tap_opaque_sent 2 threads 4 time 1310616547 total_connections 13 uptime 939 vb_active_curr_items 0 vb_active_eject 0 vb_active_ht_memory 12836864 vb_active_itm_memory 0 vb_active_num 512 vb_active_num_non_resident 0 vb_active_ops_create 0 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 0 vb_active_queue_age 0 vb_active_queue_drain 0 vb_active_queue_fill 0 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 12836864 vb_replica_itm_memory 0 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_461_gf99c147 INFO REPORT <5901.939.0> 2011-07-14 00:09:30 =============================================================================== ns_1@10.88.165.188:<5901.939.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.88.165.188', [{last_heard,{1310,616565,482660}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,30880696}, {processes,9558376}, {processes_used,9532000}, {system,21322320}, {atom,823481}, {atom_used,817012}, {binary,341256}, {code,7791057}, {ets,3028160}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1189}, {memory_data,{644874240,628781056,{<5901.943.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 15104 kB\nBuffers: 20636 kB\nCached: 358804 kB\nSwapCached: 0 kB\nActive: 275932 kB\nInactive: 261904 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 15104 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 184 kB\nWriteback: 0 kB\nAnonPages: 158432 kB\nMapped: 13208 kB\nSlab: 36496 kB\nPageTables: 3728 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 641840 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,367415296}, {buffered_memory,21131264}, {free_memory,15466496}, {total_memory,644874240}]}, {statistics, [{wall_clock,{1179023,3}}, {context_switches,{225159,0}}, {garbage_collection,{47855,159730846,0}}, {io,{{input,23129426},{output,25289360}}}, {reductions,{58581741,881032}}, {run_queue,0}, {runtime,{2390,30}}]}]}, {'ns_1@10.88.23.62', [{last_heard,{1310,616569,465327}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,33161240}, {processes,10924136}, {processes_used,10906768}, {system,22237104}, {atom,899193}, {atom_used,873961}, {binary,403320}, {code,8557947}, {ets,2975928}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {inets,"5.5.2"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1292}, {memory_data,{644874240,630751232,{<5990.234.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 13428 kB\nBuffers: 20088 kB\nCached: 362296 kB\nSwapCached: 0 kB\nActive: 276692 kB\nInactive: 262748 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 13428 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 160 kB\nWriteback: 0 kB\nAnonPages: 157056 kB\nMapped: 13212 kB\nSlab: 36568 kB\nPageTables: 3744 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 648072 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,370991104}, {buffered_memory,20570112}, {free_memory,13750272}, {total_memory,644874240}]}, {statistics, [{wall_clock,{1277501,1}}, {context_switches,{353000,0}}, {garbage_collection,{64863,261153240,0}}, {io,{{input,25814448},{output,27316309}}}, {reductions,{262280831,3034073}}, {run_queue,0}, {runtime,{8440,100}}]}]}] INFO REPORT <5901.919.0> 2011-07-14 00:09:35 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.919.0> 2011-07-14 00:10:02 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.919.0> 2011-07-14 00:10:09 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.939.0> 2011-07-14 00:10:30 =============================================================================== ns_1@10.88.165.188:<5901.939.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.88.165.188', [{last_heard,{1310,616625,482271}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,31056864}, {processes,9615504}, {processes_used,9589128}, {system,21441360}, {atom,823481}, {atom_used,817012}, {binary,413656}, {code,7791057}, {ets,3072768}]}, {system_stats,[{cpu_utilization_rate,1.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1249}, {memory_data,{644874240,629788672,{<5901.943.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 8180 kB\nBuffers: 20744 kB\nCached: 358556 kB\nSwapCached: 0 kB\nActive: 282232 kB\nInactive: 261844 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 8180 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 612 kB\nWriteback: 0 kB\nAnonPages: 164816 kB\nMapped: 13696 kB\nSlab: 36704 kB\nPageTables: 4076 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 660240 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,367161344}, {buffered_memory,21241856}, {free_memory,8376320}, {total_memory,644874240}]}, {statistics, [{wall_clock,{1239023,3}}, {context_switches,{236787,0}}, {garbage_collection,{50426,168970798,0}}, {io,{{input,23398903},{output,26157958}}}, {reductions,{62036445,864474}}, {run_queue,0}, {runtime,{2480,20}}]}]}, {'ns_1@10.88.23.62', [{last_heard,{1310,616629,465801}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,33341552}, {processes,10922272}, {processes_used,10904904}, {system,22419280}, {atom,899193}, {atom_used,873961}, {binary,433336}, {code,8557947}, {ets,3126272}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {inets,"5.5.2"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1352}, {memory_data,{644874240,631504896,{<5990.234.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 7516 kB\nBuffers: 20204 kB\nCached: 361028 kB\nSwapCached: 0 kB\nActive: 282912 kB\nInactive: 261720 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 7516 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 628 kB\nWriteback: 0 kB\nAnonPages: 163428 kB\nMapped: 13700 kB\nSlab: 36824 kB\nPageTables: 4092 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 659100 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,369692672}, {buffered_memory,20688896}, {free_memory,7696384}, {total_memory,644874240}]}, {statistics, [{wall_clock,{1337501,1}}, {context_switches,{371505,0}}, {garbage_collection,{68238,275598268,0}}, {io,{{input,26039359},{output,28275106}}}, {reductions,{278915141,5179449}}, {run_queue,0}, {runtime,{8940,170}}]}]}] INFO REPORT <5901.1085.0> 2011-07-14 00:10:48 =============================================================================== ns_1@10.88.165.188:<5901.1085.0>:stats_collector:83: Stats for bucket "default": accepting_conns 1 auth_cmds 2 auth_errors 0 bucket_active_conns 1 bucket_conns 4 bytes_read 142893 bytes_written 13951512 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 16 connection_structures 13 curr_connections 13 curr_items 0 curr_items_tot 0 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_bg_fetched 0 ep_commit_num 0 ep_commit_time 0 ep_commit_time_total 0 ep_data_age 0 ep_data_age_highwat 0 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 0 ep_dbname /opt/membase/var/lib/membase/data/default-data/default ep_dbshards 4 ep_diskqueue_drain 0 ep_diskqueue_fill 0 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 0 ep_flush_duration_total 0 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 0 ep_io_num_write 0 ep_io_read_bytes 0 ep_io_write_bytes 0 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 0 ep_kv_size 8192 ep_latency_arith_cmd 0 ep_latency_get_cmd 0 ep_latency_store_cmd 0 ep_max_data_size 515899392 ep_max_txn_size 1000 ep_mem_high_wat 386924544 ep_mem_low_wat 309539635 ep_min_data_age 0 ep_num_active_non_resident 0 ep_num_checkpoint_remover_runs200 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 26919552 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 0 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 0 ep_total_del_items 0 ep_total_enqueued 0 ep_total_new_items 0 ep_total_persisted 0 ep_uncommitted_items 0 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.0_33_ged2a54a ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 45712 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 26927744 pid 2654 pointer_size 64 rejected_conns 0 rusage_system 0.208013 rusage_user 0.792049 tap_checkpoint_start_received 512 tap_checkpoint_start_sent 512 tap_connect_received 1 tap_opaque_received 2 tap_opaque_sent 2 threads 4 time 1310616647 total_connections 13 uptime 1039 vb_active_curr_items 0 vb_active_eject 0 vb_active_ht_memory 12836864 vb_active_itm_memory 0 vb_active_num 512 vb_active_num_non_resident 0 vb_active_ops_create 0 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 0 vb_active_queue_age 0 vb_active_queue_drain 0 vb_active_queue_fill 0 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 12836864 vb_replica_itm_memory 0 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_461_gf99c147 INFO REPORT <5901.919.0> 2011-07-14 00:11:05 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.939.0> 2011-07-14 00:11:30 =============================================================================== ns_1@10.88.165.188:<5901.939.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.88.165.188', [{last_heard,{1310,616685,481976}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,30919528}, {processes,9360496}, {processes_used,9334120}, {system,21559032}, {atom,823481}, {atom_used,817012}, {binary,386152}, {code,7791057}, {ets,3216256}]}, {system_stats, [{cpu_utilization_rate,0.9900990099009901}, {swap_total,0}, {swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1309}, {memory_data,{644874240,636751872,{<5901.943.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 7188 kB\nBuffers: 20844 kB\nCached: 359312 kB\nSwapCached: 0 kB\nActive: 282436 kB\nInactive: 262636 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 7188 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 100 kB\nWriteback: 0 kB\nAnonPages: 164916 kB\nMapped: 13696 kB\nSlab: 36680 kB\nPageTables: 4088 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 652864 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,367935488}, {buffered_memory,21344256}, {free_memory,7360512}, {total_memory,644874240}]}, {statistics, [{wall_clock,{1299023,3}}, {context_switches,{248399,0}}, {garbage_collection,{53014,178270878,0}}, {io,{{input,23562519},{output,27056366}}}, {reductions,{65544364,884125}}, {run_queue,0}, {runtime,{2620,30}}]}]}, {'ns_1@10.88.23.62', [{last_heard,{1310,616689,465020}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,33227704}, {processes,10704840}, {processes_used,10687472}, {system,22522864}, {atom,899193}, {atom_used,873961}, {binary,387136}, {code,8557947}, {ets,3273104}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {inets,"5.5.2"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1412}, {memory_data,{644874240,637812736,{<5990.234.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 6408 kB\nBuffers: 20316 kB\nCached: 361796 kB\nSwapCached: 0 kB\nActive: 283664 kB\nInactive: 262524 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 6408 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 376 kB\nWriteback: 0 kB\nAnonPages: 164108 kB\nMapped: 13700 kB\nSlab: 36784 kB\nPageTables: 4100 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 659100 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,370479104}, {buffered_memory,20803584}, {free_memory,6561792}, {total_memory,644874240}]}, {statistics, [{wall_clock,{1397501,1}}, {context_switches,{390015,0}}, {garbage_collection,{71589,290113767,0}}, {io,{{input,26310383},{output,29131176}}}, {reductions,{295561225,3048837}}, {run_queue,0}, {runtime,{9390,80}}]}]}] INFO REPORT <5901.919.0> 2011-07-14 00:12:03 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.1085.0> 2011-07-14 00:12:28 =============================================================================== ns_1@10.88.165.188:<5901.1085.0>:stats_collector:83: Stats for bucket "default": accepting_conns 1 auth_cmds 2 auth_errors 0 bucket_active_conns 1 bucket_conns 4 bytes_read 149973 bytes_written 15304892 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 16 connection_structures 13 curr_connections 13 curr_items 0 curr_items_tot 0 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_bg_fetched 0 ep_commit_num 0 ep_commit_time 0 ep_commit_time_total 0 ep_data_age 0 ep_data_age_highwat 0 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 0 ep_dbname /opt/membase/var/lib/membase/data/default-data/default ep_dbshards 4 ep_diskqueue_drain 0 ep_diskqueue_fill 0 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 0 ep_flush_duration_total 0 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 0 ep_io_num_write 0 ep_io_read_bytes 0 ep_io_write_bytes 0 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 0 ep_kv_size 8192 ep_latency_arith_cmd 0 ep_latency_get_cmd 0 ep_latency_store_cmd 0 ep_max_data_size 515899392 ep_max_txn_size 1000 ep_mem_high_wat 386924544 ep_mem_low_wat 309539635 ep_min_data_age 0 ep_num_active_non_resident 0 ep_num_checkpoint_remover_runs220 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 26919552 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 0 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 0 ep_total_del_items 0 ep_total_enqueued 0 ep_total_new_items 0 ep_total_persisted 0 ep_uncommitted_items 0 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.0_33_ged2a54a ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 45712 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 26927744 pid 2654 pointer_size 64 rejected_conns 0 rusage_system 0.220013 rusage_user 0.808050 tap_checkpoint_start_received 512 tap_checkpoint_start_sent 512 tap_connect_received 1 tap_opaque_received 2 tap_opaque_sent 2 threads 4 time 1310616747 total_connections 13 uptime 1139 vb_active_curr_items 0 vb_active_eject 0 vb_active_ht_memory 12836864 vb_active_itm_memory 0 vb_active_num 512 vb_active_num_non_resident 0 vb_active_ops_create 0 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 0 vb_active_queue_age 0 vb_active_queue_drain 0 vb_active_queue_fill 0 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 12836864 vb_replica_itm_memory 0 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_461_gf99c147 INFO REPORT <5901.939.0> 2011-07-14 00:12:30 =============================================================================== ns_1@10.88.165.188:<5901.939.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.88.165.188', [{last_heard,{1310,616745,481832}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,31345664}, {processes,9704960}, {processes_used,9678584}, {system,21640704}, {atom,823481}, {atom_used,817012}, {binary,318904}, {code,7791057}, {ets,3362368}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1369}, {memory_data,{644874240,632123392,{<5901.943.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 12212 kB\nBuffers: 21056 kB\nCached: 354172 kB\nSwapCached: 0 kB\nActive: 282040 kB\nInactive: 258176 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 12212 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 192 kB\nWriteback: 0 kB\nAnonPages: 165024 kB\nMapped: 13696 kB\nSlab: 36688 kB\nPageTables: 4088 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 652864 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,362672128}, {buffered_memory,21561344}, {free_memory,12505088}, {total_memory,644874240}]}, {statistics, [{wall_clock,{1359023,3}}, {context_switches,{260690,0}}, {garbage_collection,{55781,188316772,0}}, {io,{{input,26175478},{output,30160620}}}, {reductions,{69207611,900490}}, {run_queue,0}, {runtime,{2810,20}}]}]}, {'ns_1@10.88.23.62', [{last_heard,{1310,616749,480368}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,33180200}, {processes,10519952}, {processes_used,10502584}, {system,22660248}, {atom,899193}, {atom_used,873961}, {binary,378992}, {code,8557947}, {ets,3416920}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {inets,"5.5.2"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1472}, {memory_data,{644874240,631816192,{<5990.234.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 12380 kB\nBuffers: 20540 kB\nCached: 355640 kB\nSwapCached: 0 kB\nActive: 282560 kB\nInactive: 257304 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 12380 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 208 kB\nWriteback: 0 kB\nAnonPages: 163712 kB\nMapped: 13700 kB\nSlab: 36792 kB\nPageTables: 4100 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 659100 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,364175360}, {buffered_memory,21032960}, {free_memory,12677120}, {total_memory,644874240}]}, {statistics, [{wall_clock,{1457502,2}}, {context_switches,{409283,0}}, {garbage_collection,{75146,305479879,0}}, {io,{{input,29088333},{output,32135522}}}, {reductions,{312371635,3095406}}, {run_queue,0}, {runtime,{9960,160}}]}]}] INFO REPORT <5901.919.0> 2011-07-14 00:12:55 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.919.0> 2011-07-14 00:13:18 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.939.0> 2011-07-14 00:13:30 =============================================================================== ns_1@10.88.165.188:<5901.939.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.88.165.188', [{last_heard,{1310,616805,481560}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,31608160}, {processes,9713976}, {processes_used,9687600}, {system,21894184}, {atom,823481}, {atom_used,817012}, {binary,427136}, {code,7791057}, {ets,3505808}]}, {system_stats,[{cpu_utilization_rate,2.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1429}, {memory_data,{644874240,632885248,{<5901.943.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 11336 kB\nBuffers: 21156 kB\nCached: 354932 kB\nSwapCached: 0 kB\nActive: 282264 kB\nInactive: 258960 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 11336 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 172 kB\nWriteback: 0 kB\nAnonPages: 165168 kB\nMapped: 13696 kB\nSlab: 36684 kB\nPageTables: 4092 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 652864 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,363450368}, {buffered_memory,21663744}, {free_memory,11608064}, {total_memory,644874240}]}, {statistics, [{wall_clock,{1419023,3}}, {context_switches,{272276,0}}, {garbage_collection,{58300,197502843,0}}, {io,{{input,26339749},{output,31068355}}}, {reductions,{72672064,824261}}, {run_queue,0}, {runtime,{2950,40}}]}]}, {'ns_1@10.88.23.62', [{last_heard,{1310,616809,468640}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,33250264}, {processes,10504136}, {processes_used,10486912}, {system,22746128}, {atom,899193}, {atom_used,873961}, {binary,426000}, {code,8557947}, {ets,3453784}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {inets,"5.5.2"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1532}, {memory_data,{644874240,632958976,{<5990.234.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 11636 kB\nBuffers: 20652 kB\nCached: 356412 kB\nSwapCached: 0 kB\nActive: 282808 kB\nInactive: 258092 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 11636 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 120 kB\nWriteback: 0 kB\nAnonPages: 163864 kB\nMapped: 13700 kB\nSlab: 36784 kB\nPageTables: 4104 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 659100 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,364965888}, {buffered_memory,21147648}, {free_memory,11915264}, {total_memory,644874240}]}, {statistics, [{wall_clock,{1517501,1}}, {context_switches,{427776,0}}, {garbage_collection,{78492,319914423,0}}, {io,{{input,29361817},{output,32999076}}}, {reductions,{328982966,3033112}}, {run_queue,0}, {runtime,{10450,80}}]}]}] INFO REPORT <5901.919.0> 2011-07-14 00:13:55 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.919.0> 2011-07-14 00:14:05 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.1085.0> 2011-07-14 00:14:08 =============================================================================== ns_1@10.88.165.188:<5901.1085.0>:stats_collector:83: Stats for bucket "default": accepting_conns 1 auth_cmds 2 auth_errors 0 bucket_active_conns 1 bucket_conns 4 bytes_read 157053 bytes_written 16658272 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 16 connection_structures 13 curr_connections 13 curr_items 0 curr_items_tot 0 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_bg_fetched 0 ep_commit_num 0 ep_commit_time 0 ep_commit_time_total 0 ep_data_age 0 ep_data_age_highwat 0 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 0 ep_dbname /opt/membase/var/lib/membase/data/default-data/default ep_dbshards 4 ep_diskqueue_drain 0 ep_diskqueue_fill 0 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 0 ep_flush_duration_total 0 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 0 ep_io_num_write 0 ep_io_read_bytes 0 ep_io_write_bytes 0 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 0 ep_kv_size 8192 ep_latency_arith_cmd 0 ep_latency_get_cmd 0 ep_latency_store_cmd 0 ep_max_data_size 515899392 ep_max_txn_size 1000 ep_mem_high_wat 386924544 ep_mem_low_wat 309539635 ep_min_data_age 0 ep_num_active_non_resident 0 ep_num_checkpoint_remover_runs239 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 26919552 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 0 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 0 ep_total_del_items 0 ep_total_enqueued 0 ep_total_new_items 0 ep_total_persisted 0 ep_uncommitted_items 0 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.0_33_ged2a54a ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 45712 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 26927744 pid 2654 pointer_size 64 rejected_conns 0 rusage_system 0.232014 rusage_user 0.856053 tap_checkpoint_start_received 512 tap_checkpoint_start_sent 512 tap_connect_received 1 tap_opaque_received 2 tap_opaque_sent 2 threads 4 time 1310616847 total_connections 13 uptime 1239 vb_active_curr_items 0 vb_active_eject 0 vb_active_ht_memory 12836864 vb_active_itm_memory 0 vb_active_num 512 vb_active_num_non_resident 0 vb_active_ops_create 0 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 0 vb_active_queue_age 0 vb_active_queue_drain 0 vb_active_queue_fill 0 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 12836864 vb_replica_itm_memory 0 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_461_gf99c147 INFO REPORT <5901.919.0> 2011-07-14 00:14:19 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.939.0> 2011-07-14 00:14:30 =============================================================================== ns_1@10.88.165.188:<5901.939.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.88.165.188', [{last_heard,{1310,616865,483233}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,31555056}, {processes,9692160}, {processes_used,9665784}, {system,21862896}, {atom,823481}, {atom_used,817012}, {binary,355392}, {code,7791057}, {ets,3544176}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1489}, {memory_data,{644874240,633765888,{<5901.943.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 10096 kB\nBuffers: 21264 kB\nCached: 355696 kB\nSwapCached: 0 kB\nActive: 282464 kB\nInactive: 259764 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 10096 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 488 kB\nWriteback: 0 kB\nAnonPages: 165276 kB\nMapped: 13696 kB\nSlab: 36680 kB\nPageTables: 4088 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 652864 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,364232704}, {buffered_memory,21774336}, {free_memory,10338304}, {total_memory,644874240}]}, {statistics, [{wall_clock,{1479026,5}}, {context_switches,{283933,0}}, {garbage_collection,{60830,206852286,0}}, {io,{{input,26613792},{output,31981665}}}, {reductions,{76160654,846587}}, {run_queue,0}, {runtime,{3070,10}}]}]}, {'ns_1@10.88.23.62', [{last_heard,{1310,616869,287705}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,34739464}, {processes,11211616}, {processes_used,11205680}, {system,23527848}, {atom,899193}, {atom_used,873961}, {binary,1054472}, {code,8557947}, {ets,3599496}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {inets,"5.5.2"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1592}, {memory_data,{644874240,633847808,{<5990.234.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 10272 kB\nBuffers: 20768 kB\nCached: 357192 kB\nSwapCached: 0 kB\nActive: 283012 kB\nInactive: 258876 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 10272 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 396 kB\nWriteback: 0 kB\nAnonPages: 163968 kB\nMapped: 13700 kB\nSlab: 36788 kB\nPageTables: 4104 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 659100 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,365764608}, {buffered_memory,21266432}, {free_memory,10518528}, {total_memory,644874240}]}, {statistics, [{wall_clock,{1577505,4}}, {context_switches,{446335,0}}, {garbage_collection,{81884,334489206,0}}, {io,{{input,29641955},{output,33960896}}}, {reductions,{345653537,3067907}}, {run_queue,0}, {runtime,{10950,100}}]}]}] INFO REPORT <5901.919.0> 2011-07-14 00:15:15 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.939.0> 2011-07-14 00:15:30 =============================================================================== ns_1@10.88.165.188:<5901.939.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.88.165.188', [{last_heard,{1310,616925,483024}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,31786832}, {processes,9700064}, {processes_used,9673688}, {system,22086768}, {atom,823481}, {atom_used,817012}, {binary,429240}, {code,7791057}, {ets,3690744}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1549}, {memory_data,{644874240,635170816,{<5901.943.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 9104 kB\nBuffers: 21464 kB\nCached: 356408 kB\nSwapCached: 0 kB\nActive: 282644 kB\nInactive: 260576 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 9104 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 324 kB\nWriteback: 0 kB\nAnonPages: 165384 kB\nMapped: 13696 kB\nSlab: 36712 kB\nPageTables: 4092 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 652864 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,364961792}, {buffered_memory,21979136}, {free_memory,9322496}, {total_memory,644874240}]}, {statistics, [{wall_clock,{1539024,3}}, {context_switches,{296368,0}}, {garbage_collection,{63608,216613503,0}}, {io,{{input,29271140},{output,35123479}}}, {reductions,{79763226,856519}}, {run_queue,0}, {runtime,{3280,60}}]}]}, {'ns_1@10.88.23.62', [{last_heard,{1310,616929,289150}}, {active_buckets,["default"]}, {ready_buckets,["default"]}, {replication,[{"default",1.0}]}, {memory, [{total,33212336}, {processes,10323904}, {processes_used,10306536}, {system,22888432}, {atom,899193}, {atom_used,873961}, {binary,377264}, {code,8557947}, {ets,3639368}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {inets,"5.5.2"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1652}, {memory_data,{644874240,635117568,{<5990.234.0>,1572472}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 9148 kB\nBuffers: 20976 kB\nCached: 357932 kB\nSwapCached: 0 kB\nActive: 283332 kB\nInactive: 259780 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 9148 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 212 kB\nWriteback: 0 kB\nAnonPages: 164248 kB\nMapped: 13700 kB\nSlab: 36816 kB\nPageTables: 4104 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 661048 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,366522368}, {buffered_memory,21479424}, {free_memory,9367552}, {total_memory,644874240}]}, {statistics, [{wall_clock,{1637503,2}}, {context_switches,{465619,0}}, {garbage_collection,{85463,349424986,0}}, {io,{{input,32307565},{output,37087581}}}, {reductions,{362390253,3039213}}, {run_queue,0}, {runtime,{11480,90}}]}]}] INFO REPORT <5901.919.0> 2011-07-14 00:15:38 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.1085.0> 2011-07-14 00:15:48 =============================================================================== ns_1@10.88.165.188:<5901.1085.0>:stats_collector:83: Stats for bucket "default": accepting_conns 1 auth_cmds 2 auth_errors 0 bucket_active_conns 1 bucket_conns 4 bytes_read 164133 bytes_written 18011652 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 16 connection_structures 13 curr_connections 13 curr_items 0 curr_items_tot 0 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_bg_fetched 0 ep_commit_num 0 ep_commit_time 0 ep_commit_time_total 0 ep_data_age 0 ep_data_age_highwat 0 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 0 ep_dbname /opt/membase/var/lib/membase/data/default-data/default ep_dbshards 4 ep_diskqueue_drain 0 ep_diskqueue_fill 0 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_expired 0 ep_flush_all false ep_flush_duration 0 ep_flush_duration_highwat 0 ep_flush_duration_total 0 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 0 ep_io_num_write 0 ep_io_read_bytes 0 ep_io_write_bytes 0 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 0 ep_kv_size 8192 ep_latency_arith_cmd 0 ep_latency_get_cmd 0 ep_latency_store_cmd 0 ep_max_data_size 515899392 ep_max_txn_size 1000 ep_mem_high_wat 386924544 ep_mem_low_wat 309539635 ep_min_data_age 0 ep_num_active_non_resident 0 ep_num_checkpoint_remover_runs259 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 26919552 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 0 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 0 ep_total_del_items 0 ep_total_enqueued 0 ep_total_new_items 0 ep_total_persisted 0 ep_uncommitted_items 0 ep_vb_total 1024 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.7.0_33_ged2a54a ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 45712 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.11-stable limit_maxbytes 67108864 listen_disabled_num 0 mem_used 26927744 pid 2654 pointer_size 64 rejected_conns 0 rusage_system 0.232014 rusage_user 0.892055 tap_checkpoint_start_received 512 tap_checkpoint_start_sent 512 tap_connect_received 1 tap_opaque_received 2 tap_opaque_sent 2 threads 4 time 1310616947 total_connections 13 uptime 1339 vb_active_curr_items 0 vb_active_eject 0 vb_active_ht_memory 12836864 vb_active_itm_memory 0 vb_active_num 512 vb_active_num_non_resident 0 vb_active_ops_create 0 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 0 vb_active_queue_age 0 vb_active_queue_drain 0 vb_active_queue_fill 0 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 12836864 vb_replica_itm_memory 0 vb_replica_num 512 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_461_gf99c147 INFO REPORT <5901.954.0> 2011-07-14 00:15:50 =============================================================================== ns_1@10.88.165.188:<5901.954.0>:ns_bucket_sup:82: Stopping child for dead bucket: {{per_bucket_sup, "default"}, <5901.1078.0>, supervisor, [single_bucket_sup]} INFO REPORT <5901.1078.0> 2011-07-14 00:15:50 =============================================================================== ns_1@10.88.165.188:<5901.1078.0>:single_bucket_sup:27: Delegating exit {'EXIT', <5901.955.0>, shutdown} to child supervisor: <5901.1079.0> INFO REPORT <5901.118.0> 2011-07-14 00:15:50 =============================================================================== config change: buckets -> [{configs,[]}] INFO REPORT <5901.118.0> 2011-07-14 00:15:50 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <5901.919.0> 2011-07-14 00:15:50 =============================================================================== Pushing config INFO REPORT <5901.919.0> 2011-07-14 00:15:50 =============================================================================== Pushing config done INFO REPORT <5901.123.0> 2011-07-14 00:15:50 =============================================================================== Writing isasl passwd file: "/opt/membase/var/lib/membase/data/isasl.pw" INFO REPORT <5901.1148.0> 2011-07-14 00:15:50 =============================================================================== ns_1@10.88.165.188:<5901.1148.0>:ebucketmigrator_srv:246: Got error while trying to read close ack:{error, closed} INFO REPORT <5901.1080.0> 2011-07-14 00:15:50 =============================================================================== ns_log: logging ns_memcached:2:Shutting down bucket "default" on 'ns_1@10.88.165.188' for deletion INFO REPORT <5901.1080.0> 2011-07-14 00:15:50 =============================================================================== ns_1@10.88.165.188:<5901.1080.0>:ns_storage_conf:337: Result of deleting db directory: "/opt/membase/var/lib/membase/data/default-data": ok INFO REPORT <5901.1078.0> 2011-07-14 00:15:50 =============================================================================== ns_1@10.88.165.188:<5901.1078.0>:single_bucket_sup:24: per-bucket supervisor for "default" died with reason shutdown INFO REPORT <5901.919.0> 2011-07-14 00:16:08 =============================================================================== ns_1@10.88.165.188:<5901.919.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.914.0> 2011-07-14 00:16:22 =============================================================================== ns_log: logging ns_node_disco:5:Node 'ns_1@10.88.165.188' saw that node 'ns_1@10.88.23.62' went down. INFO REPORT <5901.913.0> 2011-07-14 00:16:22 =============================================================================== ns_node_disco_log: nodes changed: ['ns_1@10.88.165.188'] INFO REPORT <0.54.0> 2011-07-14 00:18:12 =============================================================================== nonode@nohost:<0.54.0>:log_os_info:25: OS type: {unix,linux} Version: {2,6,18} Runtime info: [{otp_release,"R14B02"}, {erl_version,"5.8.3"}, {erl_version_long, "Erlang R14B02 (erts-5.8.3) [source] [64-bit] [rq:1] [async-threads:16] [hipe] [kernel-poll:false]\n"}, {system_arch_raw,"x86_64-unknown-linux-gnu"}, {system_arch,"x86_64-unknown-linux-gnu"}, {localtime,{{2011,7,14},{0,18,12}}}, {memory, [{total,14665888}, {processes,1387880}, {processes_used,1373816}, {system,13278008}, {atom,451281}, {atom_used,421728}, {binary,69400}, {code,3340681}, {ets,272616}]}, {loaded, [ns_info,log_os_info,misc,ns_log_mf_h, ns_server_cluster_sup,ns_server,timer,io_lib_fread, cpu_sup,memsup,disksup,os_mon,io_lib_pretty,io_lib_format, io_lib,io,sasl_report,release_handler,calendar,overload, alarm_handler,sasl_report_tty_h,sasl,ns_bootstrap, file_io_server,orddict,erl_eval,c,error_logger_tty_h, queue,kernel_config,user,user_sup,supervisor_bridge, standard_error,ram_file,file,beam_lib,unicode,binary,ets, gb_sets,hipe_unified_loader,packages,code_server,code, file_server,net_kernel,global_group,erl_distribution, filename,inet_gethost_native,inet_parse,inet,inet_udp,os, inet_config,inet_db,global,gb_trees,rpc,supervisor,kernel, application_master,sys,application,gen_server,erl_parse, proplists,erl_scan,lists,application_controller,proc_lib, gen,gen_event,error_logger,heart,error_handler,erlang, erl_prim_loader,prim_zip,zlib,prim_file,prim_inet,init, otp_ring0]}, {applications, [{kernel,"ERTS CXC 138 10","2.14.3"}, {stdlib,"ERTS CXC 138 10","1.17.3"}, {os_mon,"CPO CXC 138 46","2.2.5"}, {ns_server,"Membase server","1.7.1r-42-gd713cca"}, {sasl,"SASL CXC 138 11","2.1.9.3"}]}, {pre_loaded, [erlang,erl_prim_loader,prim_zip,zlib,prim_file,prim_inet, init,otp_ring0]}, {process_count,44}, {node,nonode@nohost}, {nodes,[]}, {registered, [rex,release_handler,inet_db,kernel_sup,code_server, global_name_server,overload,file_server_2, application_controller,alarm_handler,os_mon_sup, standard_error,init,cpu_sup,global_group,erl_prim_loader, sasl_sup,memsup,disksup,timer_server,standard_error_sup, user,sasl_safe_sup,error_logger,kernel_safe_sup, ns_server_cluster_sup]}, {cookie,nocookie}, {wordsize,8}, {wall_clock,0}] INFO REPORT <0.54.0> 2011-07-14 00:18:12 =============================================================================== nonode@nohost:<0.54.0>:log_os_info:27: Manifest: ["bucket_engine 1.7.0-0-g721dff0 Linux-x86_64", "ep-engine 1.7.0-33-ged2a54a Linux-x86_64", "geocouch couchbase_1.1_geo-0-gd09068e Linux-x86_64", "icu4c 2635939 Linux-x86_64","libconflate 1.7.0-0-gfee8f94 Linux-x86_64", "libmemcached 2cd40dc Linux-x86_64", "libvbucket 1.7.0-0-g4bd0aba Linux-x86_64", "manifest 1.7.0-13-gc78c994 Linux-x86_64", "membase-cli 1.7.0-1-g45a4232 Linux-x86_64", "membase-server 1.7.1r-42-gd713cca Linux-x86_64", "membasex 1.7.0-0-gf85471c Linux-x86_64", "memcached 1.4.4-461-gf99c147 Linux-x86_64", "memcachetest 0.8.3-0-g88ae3b3 Linux-x86_64", "moxi 1.7.0-4-g6a26e91 Linux-x86_64", "ns_server 1.7.0-74-gd6663ae Linux-x86_64", "otp OTP_R14B02-0-g91b2e57 Linux-x86_64", "portsigar 1.7.0-0-ga191e6c Linux-x86_64", "sigar sigar-1.6.4-406-ge1dcf32 Linux-x86_64", "spidermonkey a3c48c1 Linux-x86_64","tlm 1.7.0-8-gce9a2e8 Linux-x86_64", "vbucketmigrator 1.7.0-0-g0fdc96c Linux-x86_64"] PROGRESS REPORT <0.54.0> 2011-07-14 00:18:12 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<0.56.0>}, {name,timeout_diag_logger}, {mfargs,{timeout_diag_logger,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <0.57.0> 2011-07-14 00:18:12 =============================================================================== reading ip config from "/opt/membase/var/lib/membase/ip" PROGRESS REPORT <0.58.0> 2011-07-14 00:18:12 =============================================================================== supervisor {local,inet_gethost_native_sup} started [{pid,<0.59.0>},{mfa,{inet_gethost_native,init,[[]]}}] PROGRESS REPORT <0.26.0> 2011-07-14 00:18:12 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<0.58.0>}, {name,inet_gethost_native_sup}, {mfargs,{inet_gethost_native,start_link,[]}}, {restart_type,temporary}, {shutdown,1000}, {child_type,worker}] INFO REPORT <0.57.0> 2011-07-14 00:18:12 =============================================================================== nonode@nohost:<0.57.0>:dist_manager:105: Attempting to bring up net_kernel with name 'ns_1@10.88.165.188' PROGRESS REPORT <0.60.0> 2011-07-14 00:18:12 =============================================================================== supervisor {local,net_sup} started [{pid,<0.61.0>}, {name,erl_epmd}, {mfargs,{erl_epmd,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] PROGRESS REPORT <0.60.0> 2011-07-14 00:18:12 =============================================================================== supervisor {local,net_sup} started [{pid,<0.62.0>}, {name,auth}, {mfargs,{auth,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] PROGRESS REPORT <5901.60.0> 2011-07-14 00:18:12 =============================================================================== supervisor {local,net_sup} started [{pid,<5901.63.0>}, {name,net_kernel}, {mfargs,{net_kernel,start_link,[['ns_1@10.88.165.188',longnames]]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] PROGRESS REPORT <5901.11.0> 2011-07-14 00:18:12 =============================================================================== supervisor {local,kernel_sup} started [{pid,<5901.60.0>}, {name,net_sup_dynamic}, {mfargs,{erl_distribution,start_link, [['ns_1@10.88.165.188',longnames]]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,supervisor}] PROGRESS REPORT <5901.54.0> 2011-07-14 00:18:12 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<5901.57.0>}, {name,dist_manager}, {mfargs,{dist_manager,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.54.0> 2011-07-14 00:18:12 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<5901.66.0>}, {name,ns_cluster}, {mfargs,{ns_cluster,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <5901.67.0> 2011-07-14 00:18:12 =============================================================================== supervisor {local,mb_mnesia_sup} started [{pid,<5901.68.0>}, {name,mb_mnesia_events}, {mfargs,{gen_event,start_link,[{local,mb_mnesia_events}]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <5901.74.0> 2011-07-14 00:18:12 =============================================================================== supervisor {local,mnesia_sup} started [{pid,<5901.75.0>}, {name,mnesia_event}, {mfargs,{mnesia_sup,start_event,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] PROGRESS REPORT <5901.76.0> 2011-07-14 00:18:12 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.77.0>}, {name,mnesia_monitor}, {mfargs,{mnesia_monitor,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <5901.76.0> 2011-07-14 00:18:12 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.78.0>}, {name,mnesia_subscr}, {mfargs,{mnesia_subscr,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <5901.76.0> 2011-07-14 00:18:12 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.79.0>}, {name,mnesia_locker}, {mfargs,{mnesia_locker,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <5901.76.0> 2011-07-14 00:18:12 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.80.0>}, {name,mnesia_recover}, {mfargs,{mnesia_recover,start,[]}}, {restart_type,permanent}, {shutdown,180000}, {child_type,worker}] PROGRESS REPORT <5901.26.0> 2011-07-14 00:18:12 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<5901.82.0>}, {name,dets_sup}, {mfargs,{dets_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,supervisor}] PROGRESS REPORT <5901.26.0> 2011-07-14 00:18:12 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<5901.83.0>}, {name,dets}, {mfargs,{dets_server,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] PROGRESS REPORT <5901.26.0> 2011-07-14 00:18:12 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<5901.87.0>}, {name,disk_log_sup}, {mfargs,{disk_log_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,supervisor}] PROGRESS REPORT <5901.26.0> 2011-07-14 00:18:12 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<5901.88.0>}, {name,disk_log_server}, {mfargs,{disk_log_server,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] PROGRESS REPORT <5901.76.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.81.0>}, {name,mnesia_tm}, {mfargs,{mnesia_tm,start,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] PROGRESS REPORT <5901.76.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.113.0>}, {name,mnesia_checkpoint_sup}, {mfargs,{mnesia_checkpoint_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.76.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.114.0>}, {name,mnesia_snmp_sup}, {mfargs,{mnesia_snmp_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.76.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.115.0>}, {name,mnesia_controller}, {mfargs,{mnesia_controller,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <5901.76.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<5901.116.0>}, {name,mnesia_late_loader}, {mfargs,{mnesia_late_loader,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <5901.74.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,mnesia_sup} started [{pid,<5901.76.0>}, {name,mnesia_kernel_sup}, {mfargs,{mnesia_kernel_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.7.0> 2011-07-14 00:18:13 =============================================================================== application mnesia started_at 'ns_1@10.88.165.188' INFO REPORT <5901.69.0> 2011-07-14 00:18:13 =============================================================================== ns_1@10.88.165.188:<5901.69.0>:mb_mnesia:417: Using existing disk schema on ['ns_1@10.88.165.188']. INFO REPORT <5901.69.0> 2011-07-14 00:18:13 =============================================================================== ns_1@10.88.165.188:<5901.69.0>:mb_mnesia:427: Have local copy of 'stats_archiver-default-day' INFO REPORT <5901.69.0> 2011-07-14 00:18:13 =============================================================================== ns_1@10.88.165.188:<5901.69.0>:mb_mnesia:427: Have local copy of local_config INFO REPORT <5901.69.0> 2011-07-14 00:18:13 =============================================================================== ns_1@10.88.165.188:<5901.69.0>:mb_mnesia:427: Have local copy of 'stats_archiver-@system-day' INFO REPORT <5901.69.0> 2011-07-14 00:18:13 =============================================================================== ns_1@10.88.165.188:<5901.69.0>:mb_mnesia:427: Have local copy of 'stats_archiver-default-month' INFO REPORT <5901.69.0> 2011-07-14 00:18:13 =============================================================================== ns_1@10.88.165.188:<5901.69.0>:mb_mnesia:427: Have local copy of cluster INFO REPORT <5901.69.0> 2011-07-14 00:18:13 =============================================================================== ns_1@10.88.165.188:<5901.69.0>:mb_mnesia:427: Have local copy of 'stats_archiver-default-week' INFO REPORT <5901.69.0> 2011-07-14 00:18:13 =============================================================================== ns_1@10.88.165.188:<5901.69.0>:mb_mnesia:427: Have local copy of 'stats_archiver-@system-week' INFO REPORT <5901.69.0> 2011-07-14 00:18:13 =============================================================================== ns_1@10.88.165.188:<5901.69.0>:mb_mnesia:427: Have local copy of 'stats_archiver-@system-minute' INFO REPORT <5901.69.0> 2011-07-14 00:18:13 =============================================================================== ns_1@10.88.165.188:<5901.69.0>:mb_mnesia:427: Have local copy of 'stats_archiver-default-year' INFO REPORT <5901.69.0> 2011-07-14 00:18:13 =============================================================================== ns_1@10.88.165.188:<5901.69.0>:mb_mnesia:427: Have local copy of 'stats_archiver-@system-year' INFO REPORT <5901.69.0> 2011-07-14 00:18:13 =============================================================================== ns_1@10.88.165.188:<5901.69.0>:mb_mnesia:427: Have local copy of 'stats_archiver-@system-month' INFO REPORT <5901.69.0> 2011-07-14 00:18:13 =============================================================================== ns_1@10.88.165.188:<5901.69.0>:mb_mnesia:427: Have local copy of 'stats_archiver-default-minute' INFO REPORT <5901.69.0> 2011-07-14 00:18:13 =============================================================================== ns_1@10.88.165.188:<5901.69.0>:mb_mnesia:427: Have local copy of 'stats_archiver-default-hour' INFO REPORT <5901.69.0> 2011-07-14 00:18:13 =============================================================================== ns_1@10.88.165.188:<5901.69.0>:mb_mnesia:427: Have local copy of 'stats_archiver-@system-hour' INFO REPORT <5901.69.0> 2011-07-14 00:18:13 =============================================================================== ns_1@10.88.165.188:<5901.69.0>:mb_mnesia:245: Current config: [{access_module,mnesia}, {auto_repair,true}, {backup_module,mnesia_backup}, {checkpoints,[]}, {db_nodes,['ns_1@10.88.165.188']}, {debug,verbose}, {directory,"/opt/membase/var/lib/membase/mnesia"}, {dump_log_load_regulation,false}, {dump_log_time_threshold,180000}, {dump_log_update_in_place,true}, {dump_log_write_threshold,1000}, {embedded_mnemosyne,false}, {event_module,mnesia_event}, {extra_db_nodes,[]}, {fallback_activated,false}, {held_locks,[]}, {ignore_fallback_at_startup,false}, {fallback_error_function,{mnesia,lkill}}, {is_running,yes}, {local_tables,['stats_archiver-default-week','stats_archiver-default-year', 'stats_archiver-default-month', 'stats_archiver-default-minute','stats_archiver-@system-week', 'stats_archiver-default-day','stats_archiver-@system-year', 'stats_archiver-@system-month','stats_archiver-@system-day', 'stats_archiver-@system-minute',local_config, 'stats_archiver-default-hour','stats_archiver-@system-hour', cluster,schema]}, {lock_queue,[]}, {log_version,"4.3"}, {master_node_tables,[]}, {max_wait_for_decision,10000}, {protocol_version,{7,6}}, {running_db_nodes,['ns_1@10.88.165.188']}, {schema_location,opt_disc}, {schema_version,{3,0}}, {subscribers,[<5901.75.0>,<5901.69.0>]}, {tables,['stats_archiver-default-day',local_config, 'stats_archiver-@system-day','stats_archiver-default-month',cluster, 'stats_archiver-default-week','stats_archiver-@system-week', 'stats_archiver-@system-minute','stats_archiver-default-year', 'stats_archiver-@system-year',schema,'stats_archiver-@system-month', 'stats_archiver-default-minute','stats_archiver-default-hour', 'stats_archiver-@system-hour']}, {transaction_commits,2}, {transaction_failures,14}, {transaction_log_writes,0}, {transaction_restarts,0}, {transactions,[]}, {use_dir,true}, {core_dir,false}, {no_table_loaders,2}, {dc_dump_limit,4}, {send_compressed,0}, {version,"4.4.17"}] Peers: ['ns_1@10.88.165.188'] PROGRESS REPORT <5901.67.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,mb_mnesia_sup} started [{pid,<5901.69.0>}, {name,mb_mnesia}, {mfargs,{mb_mnesia,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <5901.54.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<5901.67.0>}, {name,mb_mnesia_sup}, {mfargs,{mb_mnesia_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <5901.218.0> 2011-07-14 00:18:13 =============================================================================== loading static ns_config from "/opt/membase/etc/membase/config" PROGRESS REPORT <5901.218.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_config_sup} started [{pid,<5901.219.0>}, {name,ns_config_events}, {mfargs,{gen_event,start_link,[{local,ns_config_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.218.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_config_sup} started [{pid,<5901.220.0>}, {name,ns_config}, {mfargs,{ns_config,start_link, ["/opt/membase/etc/membase/config", ns_config_default]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.218.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_config_sup} started [{pid,<5901.222.0>}, {name,ns_config_remote}, {mfargs,{ns_config_replica,start_link,[{local,ns_config_remote}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.223.0> 2011-07-14 00:18:13 =============================================================================== isasl_sync init: ["/opt/membase/var/lib/membase/data/isasl.pw","_admin", "_admin"] INFO REPORT <5901.223.0> 2011-07-14 00:18:13 =============================================================================== isasl_sync init buckets: [] INFO REPORT <5901.223.0> 2011-07-14 00:18:13 =============================================================================== Writing isasl passwd file: "/opt/membase/var/lib/membase/data/isasl.pw" PROGRESS REPORT <5901.218.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_config_sup} started [{pid,<5901.223.0>}, {name,ns_config_isasl_sync}, {mfargs,{ns_config_isasl_sync,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.218.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_config_sup} started [{pid,<5901.225.0>}, {name,ns_config_log}, {mfargs,{ns_config_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.54.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<5901.218.0>}, {name,ns_config_sup}, {mfargs,{ns_config_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.226.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.227.0>}, {name,ns_log}, {mfargs,{ns_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.226.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.228.0>}, {name,ns_log_events}, {mfargs,{gen_event,start_link,[{local,ns_log_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.229.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<5901.230.0>}, {name,ns_node_disco_events}, {mfargs,{gen_event,start_link,[{local,ns_node_disco_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.231.0> 2011-07-14 00:18:13 =============================================================================== Initting ns_node_disco with [] INFO REPORT <5901.232.0> 2011-07-14 00:18:13 =============================================================================== ns_node_disco cookie_sync INFO REPORT <5901.232.0> 2011-07-14 00:18:13 =============================================================================== ns_log: logging ns_node_disco:2:Node 'ns_1@10.88.165.188' synchronized otp cookie brohpzslpcmrmtnv from cluster INFO REPORT <5901.232.0> 2011-07-14 00:18:13 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.88.165.188','ns_1@10.88.23.62'], with cookie: brohpzslpcmrmtnv INFO REPORT <5901.232.0> 2011-07-14 00:18:13 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.88.165.188','ns_1@10.88.23.62'], with cookie: brohpzslpcmrmtnv PROGRESS REPORT <5901.229.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<5901.231.0>}, {name,ns_node_disco}, {mfargs,{ns_node_disco,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.229.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<5901.237.0>}, {name,ns_node_disco_log}, {mfargs,{ns_node_disco_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.229.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<5901.238.0>}, {name,ns_node_disco_conf_events}, {mfargs,{ns_node_disco_conf_events,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.239.0> 2011-07-14 00:18:13 =============================================================================== ns_1@10.88.165.188:<5901.239.0>:ns_config_rep:56: init pulling INFO REPORT <5901.239.0> 2011-07-14 00:18:13 =============================================================================== ns_1@10.88.165.188:<5901.239.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== config change: buckets -> [{configs,[]}] INFO REPORT <5901.239.0> 2011-07-14 00:18:13 =============================================================================== ns_1@10.88.165.188:<5901.239.0>:ns_config_rep:58: init pushing INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== config change: rebalance_status -> none INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== config change: {node,'ns_1@10.88.165.188',membership} -> active INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <5901.239.0> 2011-07-14 00:18:13 =============================================================================== ns_1@10.88.165.188:<5901.239.0>:ns_config_rep:62: init reannouncing INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== config change: auto_failover_cfg -> [{enabled,false},{timeout,30},{max_nodes,1},{count,0}] INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== config change: buckets -> [{configs,[]}] INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== config change: email_alerts -> [{recipients,["root@localhost"]}, {sender,"membase@localhost"}, {enabled,true}, {email_server,[{user,[]}, {pass,[]}, {host,"localhost"}, {port,25}, {encrypt,false}]}, {alerts,[auto_failover_node,auto_failover_maximum_reached, auto_failover_other_nodes_down,auto_failover_cluster_too_small]}] INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== config change: memory_quota -> 492 INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== ns_node_disco_conf_events config on nodes_wanted INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== config change: nodes_wanted -> ['ns_1@10.88.165.188','ns_1@10.88.23.62'] INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== ns_node_disco_conf_events config on otp INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== config change: otp -> [{cookie,brohpzslpcmrmtnv}] INFO REPORT <5901.244.0> 2011-07-14 00:18:13 =============================================================================== ns_node_disco cookie_sync INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== config change: port_servers -> [{moxi,"/opt/membase/bin/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{rest,port}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",{"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD",{"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,exit_status,port_server_send_eol,stderr_to_stdout,stream]}, {memcached,"/opt/membase/bin/memcached", ["-X","/opt/membase/lib/memcached/stdin_term_handler.so","-p", {"~B",[port]}, "-E","/opt/membase/lib/memcached/bucket_engine.so","-B","binary", "-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status,port_server_send_eol, stream]}] INFO REPORT <5901.245.0> 2011-07-14 00:18:13 =============================================================================== ns_node_disco cookie_sync INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== config change: rebalance_status -> none INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== config change: replication -> [{enabled,true}] INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== config change: rest -> [{port,8091}] INFO REPORT <5901.244.0> 2011-07-14 00:18:13 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.88.165.188','ns_1@10.88.23.62'], with cookie: brohpzslpcmrmtnv INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== config change: rest_creds -> ******** INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== config change: uuid -> <<"78161b79-ffa7-4001-15e8-0cc7000003af">> INFO REPORT <5901.245.0> 2011-07-14 00:18:13 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.88.165.188','ns_1@10.88.23.62'], with cookie: brohpzslpcmrmtnv INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== config change: {node,'ns_1@10.88.165.188',config_version} -> {1,7,1} INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== config change: {node,'ns_1@10.88.165.188',isasl} -> [{path,"/opt/membase/var/lib/membase/data/isasl.pw"}] INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== config change: {node,'ns_1@10.88.165.188',membership} -> active INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== config change: {node,'ns_1@10.88.165.188',memcached} -> [{port,11210}, {dbdir,"/opt/membase/var/lib/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"/opt/membase/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"/opt/membase/lib/memcached/ep.so"}, {initfile,"/opt/membase/etc/membase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine,"/opt/membase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}] INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== config change: {node,'ns_1@10.88.165.188',moxi} -> [{port,11211},{verbosity,[]}] INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== config change: {node,'ns_1@10.88.165.188',ns_log} -> [{filename,"/opt/membase/var/lib/membase/data/ns_log"}] INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== config change: {node,'ns_1@10.88.165.188',rest} -> [{port,8091}] INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== config change: {node,'ns_1@10.88.23.62',config_version} -> {1,7,1} INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== config change: {node,'ns_1@10.88.23.62',isasl} -> [{path,"/opt/membase/var/lib/membase/data/isasl.pw"}] INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== config change: {node,'ns_1@10.88.23.62',membership} -> active INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== config change: {node,'ns_1@10.88.23.62',memcached} -> [{port,11210}, {dbdir,"/opt/membase/var/lib/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"/opt/membase/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"/opt/membase/lib/memcached/ep.so"}, {initfile,"/opt/membase/etc/membase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine,"/opt/membase/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}] INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== config change: {node,'ns_1@10.88.23.62',moxi} -> [{port,11211},{verbosity,[]}] INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== config change: {node,'ns_1@10.88.23.62',ns_log} -> [{filename,"/opt/membase/var/lib/membase/data/ns_log"}] INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== config change: {node,'ns_1@10.88.23.62',rest} -> [{port,8091}] INFO REPORT <5901.219.0> 2011-07-14 00:18:13 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <5901.239.0> 2011-07-14 00:18:13 =============================================================================== Pushing config PROGRESS REPORT <5901.229.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<5901.239.0>}, {name,ns_config_rep}, {mfargs,{ns_config_rep,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.226.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.229.0>}, {name,ns_node_disco_sup}, {mfargs,{ns_node_disco_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.226.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.247.0>}, {name,ns_tick_event}, {mfargs,{gen_event,start_link,[{local,ns_tick_event}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.239.0> 2011-07-14 00:18:13 =============================================================================== Pushing config done INFO REPORT <5901.250.0> 2011-07-14 00:18:13 =============================================================================== ns_1@10.88.165.188:<5901.250.0>:mb_master:94: Starting as candidate. Peers: ['ns_1@10.88.165.188', 'ns_1@10.88.23.62'] PROGRESS REPORT <5901.226.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.250.0>}, {name,mb_master}, {mfargs,{mb_master,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.226.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.251.0>}, {name,buckets_events}, {mfargs,{gen_event,start_link,[{local,buckets_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.252.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_mail_sup} started [{pid,<5901.253.0>}, {name,ns_mail}, {mfargs,{ns_mail,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.252.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_mail_sup} started [{pid,<5901.254.0>}, {name,ns_mail_log}, {mfargs,{ns_mail_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.226.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.252.0>}, {name,ns_mail_sup}, {mfargs,{ns_mail_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <5901.228.0> 2011-07-14 00:18:13 =============================================================================== ns_1@10.88.165.188:<5901.228.0>:ns_mail_log:45: ns_mail_log started up PROGRESS REPORT <5901.226.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.255.0>}, {name,ns_heart}, {mfargs,{ns_heart,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.226.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.256.0>}, {name,ns_doctor}, {mfargs,{ns_doctor,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.258.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<5901.259.0>}, {name,menelaus_web}, {mfargs,{menelaus_web,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <5901.258.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<5901.261.0>}, {name,menelaus_event}, {mfargs,{menelaus_event,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <5901.258.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<5901.262.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <5901.258.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<5901.263.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <5901.226.0> 2011-07-14 00:18:13 =============================================================================== ns_log: logging menelaus_sup:1:Membase Server has started on web port 8091 on node 'ns_1@10.88.165.188'. PROGRESS REPORT <5901.226.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.258.0>}, {name,menelaus}, {mfargs,{menelaus_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.264.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<5901.265.0>}, {name,ns_port_init}, {mfargs,{ns_port_init,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <5901.266.0> 2011-07-14 00:18:13 =============================================================================== starting ns_port_server with delay of 5000 INFO REPORT <5901.244.0> 2011-07-14 00:18:13 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.88.165.188','ns_1@10.88.23.62'], with cookie: brohpzslpcmrmtnv INFO REPORT <5901.245.0> 2011-07-14 00:18:13 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.88.165.188','ns_1@10.88.23.62'], with cookie: brohpzslpcmrmtnv PROGRESS REPORT <5901.264.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<5901.266.0>}, {name, {moxi,"/opt/membase/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]}}, {mfargs, {supervisor_cushion,start_link, [moxi,5000,ns_port_server,start_link, [moxi,"/opt/membase/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]]]}}, {restart_type,permanent}, {shutdown,10000}, {child_type,worker}] INFO REPORT <5901.268.0> 2011-07-14 00:18:13 =============================================================================== starting ns_port_server with delay of 5000 PROGRESS REPORT <5901.264.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<5901.268.0>}, {name, {memcached,"/opt/membase/bin/memcached", ["-X","/opt/membase/lib/memcached/stdin_term_handler.so", "-p","11210","-E", "/opt/membase/lib/memcached/bucket_engine.so","-B", "binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE", "/opt/membase/var/lib/membase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status, port_server_send_eol,stream]}}, {mfargs, {supervisor_cushion,start_link, [memcached,5000,ns_port_server,start_link, [memcached,"/opt/membase/bin/memcached", ["-X","/opt/membase/lib/memcached/stdin_term_handler.so", "-p","11210","-E", "/opt/membase/lib/memcached/bucket_engine.so","-B", "binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE", "/opt/membase/var/lib/membase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status, port_server_send_eol,stream]]]}}, {restart_type,permanent}, {shutdown,10000}, {child_type,worker}] PROGRESS REPORT <5901.226.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.264.0>}, {name,ns_port_sup}, {mfargs,{ns_port_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,60000}, {child_type,worker}] PROGRESS REPORT <5901.226.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.270.0>}, {name,ns_stats_event}, {mfargs,{gen_event,start_link,[{local,ns_stats_event}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.226.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.272.0>}, {name,ns_bucket_worker}, {mfargs,{work_queue,start_link,[ns_bucket_worker]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.226.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.273.0>}, {name,ns_bucket_sup}, {mfargs,{ns_bucket_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] ERROR REPORT <5901.255.0> 2011-07-14 00:18:13 =============================================================================== ns_1@10.88.165.188:<5901.255.0>:ns_heart:137: Failed to grab system stats: {'EXIT',{noproc,{gen_server,call, [{'stats_reader-@system','ns_1@10.88.165.188'}, {latest,"minute"}]}}} PROGRESS REPORT <5901.226.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.274.0>}, {name,system_stats_collector}, {mfargs,{system_stats_collector,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.226.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.276.0>}, {name,{stats_archiver,"@system"}}, {mfargs,{stats_archiver,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.226.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.277.0>}, {name,{stats_reader,"@system"}}, {mfargs,{stats_reader,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.226.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.278.0>}, {name,ns_moxi_sup_work_queue}, {mfargs,{work_queue,start_link,[ns_moxi_sup_work_queue]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.226.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<5901.279.0>}, {name,ns_moxi_sup}, {mfargs,{ns_moxi_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.54.0> 2011-07-14 00:18:13 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<5901.226.0>}, {name,ns_server_sup}, {mfargs,{ns_server_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.7.0> 2011-07-14 00:18:13 =============================================================================== application ns_server started_at 'ns_1@10.88.165.188' INFO REPORT <5901.256.0> 2011-07-14 00:18:13 =============================================================================== ns_1@10.88.165.188:<5901.256.0>:ns_doctor:82: Got initial status [{'ns_1@10.88.165.188', [{last_heard, {1310, 617093, 946891}}, {active_buckets, []}, {ready_buckets, []}, {replication, []}, {memory, [{total, 25836856}, {processes, 5040504}, {processes_used, 5024648}, {system, 20796352}, {atom, 769233}, {atom_used, 744337}, {binary, 553888}, {code, 6870389}, {ets, 3320864}]}, {system_stats, [{cpu_utilization_rate, 0}, {swap_total, 0}, {swap_used, 0}]}, {interesting_stats, []}, {cluster_compatibility_version, 1}, {version, [{kernel, "2.14.3"}, {mnesia, "4.4.17"}, {stdlib, "1.17.3"}, {os_mon, "2.2.5"}, {ns_server, "1.7.1r-42-gd713cca"}, {sasl, "2.1.9.3"}]}, {system_arch, "x86_64-unknown-linux-gnu"}, {wall_clock, 1}, {memory_data, {644874240, 558825472, {<5901.7.0>, 196872}}}, {disk_data, [{"/", 5904748, 66}, {"/dev/shm", 314880, 0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 71532 kB\nBuffers: 21920 kB\nCached: 363876 kB\nSwapCached: 0 kB\nActive: 212748 kB\nInactive: 269208 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 71532 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 11852 kB\nWriteback: 20 kB\nAnonPages: 96140 kB\nMapped: 11588 kB\nSlab: 36700 kB\nPageTables: 3720 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 503472 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory, 644874240}, {free_swap, 0}, {total_swap, 0}, {cached_memory, 372609024}, {buffered_memory, 22446080}, {free_memory, 74125312}, {total_memory, 644874240}]}, {statistics, [{wall_clock, {1679, 65}}, {context_switches, {10077, 0}}, {garbage_collection, {2041, 4330574, 0}}, {io, {{input, 14405676}, {output, 5098305}}}, {reductions, {1560639, 89396}}, {run_queue, 0}, {runtime, {380, 20}}]}]}, {'ns_1@10.88.23.62', [{last_heard, {1310, 617093, 946905}}, {active_buckets, []}, {ready_buckets, []}, {replication, []}, {memory, [{total, 26663344}, {processes, 5622032}, {processes_used, 5606256}, {system, 21041312}, {atom, 782969}, {atom_used, 764039}, {binary, 398832}, {code, 7228817}, {ets, 3352528}]}, {system_stats, [{cpu_utilization_rate, 0.0}, {swap_total, 0}, {swap_used, 0}]}, {interesting_stats, []}, {cluster_compatibility_version, 1}, {version, [{kernel, "2.14.3"}, {mnesia, "4.4.17"}, {stdlib, "1.17.3"}, {os_mon, "2.2.5"}, {ns_server, "1.7.1r-42-gd713cca"}, {sasl, "2.1.9.3"}]}, {system_arch, "x86_64-unknown-linux-gnu"}, {wall_clock, 5}, {memory_data, {644874240, 560467968, {<5990.7.0>, 196872}}}, {disk_data, [{"/", 5904748, 66}, {"/dev/shm", 314880, 0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 73072 kB\nBuffers: 21440 kB\nCached: 365560 kB\nSwapCached: 0 kB\nActive: 212552 kB\nInactive: 268172 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 73072 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 496 kB\nWriteback: 0 kB\nAnonPages: 93748 kB\nMapped: 11124 kB\nSlab: 36692 kB\nPageTables: 3552 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 397384 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory, 644874240}, {free_swap, 0}, {total_swap, 0}, {cached_memory, 374333440}, {buffered_memory, 21954560}, {free_memory, 74842112}, {total_memory, 644874240}]}, {statistics, [{wall_clock, {1520, 1291}}, {context_switches, {9960, 0}}, {garbage_collection, {2037, 4293068, 0}}, {io, {{input, 14416581}, {output, 5110271}}}, {reductions, {1548852, 1548852}}, {run_queue, 0}, {runtime, {440, 440}}]}]}] INFO REPORT <5901.267.0> 2011-07-14 00:18:14 =============================================================================== moxi<0.267.0>: 2011-07-14 00:18:13: (cproxy_config.c.317) env: MOXI_SASL_PLAIN_USR (13) moxi<0.267.0>: 2011-07-14 00:18:13: (cproxy_config.c.326) env: MOXI_SASL_PLAIN_PWD (8) INFO REPORT <5901.250.0> 2011-07-14 00:18:23 =============================================================================== ns_1@10.88.165.188:<5901.250.0>:mb_master:116: Haven't heard from a higher priority node or a master, so I'm taking over. INFO REPORT <5901.303.0> 2011-07-14 00:18:23 =============================================================================== ns_1@10.88.165.188:<5901.303.0>:misc:784: start_singleton(gen_fsm, ns_orchestrator, [], []): started as <5901.304.0> on 'ns_1@10.88.165.188' PROGRESS REPORT <5901.303.0> 2011-07-14 00:18:23 =============================================================================== supervisor {local,mb_master_sup} started [{pid,<5901.304.0>}, {name,ns_orchestrator}, {mfargs,{ns_orchestrator,start_link,[]}}, {restart_type,permanent}, {shutdown,20}, {child_type,worker}] INFO REPORT <5901.303.0> 2011-07-14 00:18:23 =============================================================================== ns_1@10.88.165.188:<5901.303.0>:misc:784: start_singleton(gen_server, ns_tick, [], []): started as <5901.308.0> on 'ns_1@10.88.165.188' PROGRESS REPORT <5901.303.0> 2011-07-14 00:18:23 =============================================================================== supervisor {local,mb_master_sup} started [{pid,<5901.308.0>}, {name,ns_tick}, {mfargs,{ns_tick,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <5901.309.0> 2011-07-14 00:18:23 =============================================================================== ns_1@10.88.165.188:<5901.309.0>:auto_failover:120: init auto_failover. INFO REPORT <5901.303.0> 2011-07-14 00:18:23 =============================================================================== ns_1@10.88.165.188:<5901.303.0>:misc:784: start_singleton(gen_server, auto_failover, [], []): started as <5901.309.0> on 'ns_1@10.88.165.188' PROGRESS REPORT <5901.303.0> 2011-07-14 00:18:23 =============================================================================== supervisor {local,mb_master_sup} started [{pid,<5901.309.0>}, {name,auto_failover}, {mfargs,{auto_failover,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <5901.219.0> 2011-07-14 00:18:32 =============================================================================== config change: buckets -> [{configs,[{"default", [{num_replicas,1}, {ram_quota,515899392}, {auth_type,sasl}, {sasl_password,[]}, {type,membase}, {num_vbuckets,1024}, {servers,[]}]}]}] INFO REPORT <5901.219.0> 2011-07-14 00:18:32 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <5901.239.0> 2011-07-14 00:18:32 =============================================================================== Pushing config INFO REPORT <5901.239.0> 2011-07-14 00:18:32 =============================================================================== Pushing config done INFO REPORT <5901.223.0> 2011-07-14 00:18:32 =============================================================================== Writing isasl passwd file: "/opt/membase/var/lib/membase/data/isasl.pw" INFO REPORT <5901.272.0> 2011-07-14 00:18:32 =============================================================================== ns_1@10.88.165.188:<5901.272.0>:ns_bucket_sup:75: Starting new child: {{per_bucket_sup, "default"}, {single_bucket_sup, start_link, ["default"]}, permanent, infinity, supervisor, [single_bucket_sup]} INFO REPORT <5901.219.0> 2011-07-14 00:18:32 =============================================================================== config change: buckets -> [{configs,[{"default", [{num_replicas,1}, {ram_quota,515899392}, {auth_type,sasl}, {sasl_password,[]}, {type,membase}, {num_vbuckets,1024}, {servers,['ns_1@10.88.165.188','ns_1@10.88.23.62']}]}]}] INFO REPORT <5901.219.0> 2011-07-14 00:18:32 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <5901.239.0> 2011-07-14 00:18:32 =============================================================================== Pushing config INFO REPORT <5901.239.0> 2011-07-14 00:18:32 =============================================================================== Pushing config done INFO REPORT <5901.334.0> 2011-07-14 00:18:32 =============================================================================== ns_1@10.88.165.188:<5901.334.0>:ns_janitor:302: Waiting for "default" on ['ns_1@10.88.165.188', 'ns_1@10.88.23.62'] PROGRESS REPORT <5901.273.0> 2011-07-14 00:18:32 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<5901.341.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.343.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.345.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.346.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.347.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.348.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.349.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.343.0> 2011-07-14 00:18:32 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.343.0> 2011-07-14 00:18:32 =============================================================================== ** Generic server <5901.343.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.1785>}, init, {1310,617112,246691}, "default",#Port<5901.2406>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.343.0> 2011-07-14 00:18:32 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.343.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.341.0>] messages [] links [<5901.49.0>,<5901.342.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7220 SUPERVISOR REPORT <5901.342.0> 2011-07-14 00:18:32 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.343.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.359.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.360.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.361.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.362.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.363.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.364.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.359.0> 2011-07-14 00:18:32 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.359.0> 2011-07-14 00:18:32 =============================================================================== ** Generic server <5901.359.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.2011>}, init, {1310,617112,746749}, "default",#Port<5901.2449>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.359.0> 2011-07-14 00:18:32 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.359.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.341.0>] messages [] links [<5901.49.0>,<5901.342.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6653 SUPERVISOR REPORT <5901.342.0> 2011-07-14 00:18:32 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.359.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.361.0> 2011-07-14 00:18:32 =============================================================================== ns_1@10.88.165.188:<5901.361.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.365.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.366.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.367.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.368.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.369.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.370.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.267.0> 2011-07-14 00:18:33 =============================================================================== moxi<0.267.0>: 2011-07-14 00:18:34: (agent_config.c.705) ERROR: bad JSON configuration from http://127.0.0.1:8091/pools/default/saslBucketsStreaming: Empty serverList ({ moxi<0.267.0>: "name": "default", moxi<0.267.0>: "nodeLocator": "vbucket", moxi<0.267.0>: "saslPassword": "", moxi<0.267.0>: "nodes": [], moxi<0.267.0>: "vBucketServerMap": { moxi<0.267.0>: "hashAlgorithm": "CRC", moxi<0.267.0>: "numReplicas": 1, moxi<0.267.0>: "serverList": [], moxi<0.267.0>: "vBucketMap": [] ERROR REPORT <5901.267.0> 2011-07-14 00:18:33 =============================================================================== ns_1@10.88.165.188:<5901.267.0>:ns_port_server:161: Dropped 104 log lines from moxi INFO REPORT <5901.365.0> 2011-07-14 00:18:33 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.365.0> 2011-07-14 00:18:33 =============================================================================== ** Generic server <5901.365.0> terminating ** Last message in was list_vbuckets_prevstate ** When Server state == {state,{interval,#Ref<5901.0.0.2071>}, init, {1310,617112,903050}, "default",#Port<5901.2450>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.365.0> 2011-07-14 00:18:33 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.365.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.341.0>] messages [] links [<5901.49.0>,<5901.342.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6665 SUPERVISOR REPORT <5901.342.0> 2011-07-14 00:18:33 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.365.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker CRASH REPORT <5901.334.0> 2011-07-14 00:18:33 =============================================================================== Crashing process initial_call {ns_janitor,cleanup,['Argument__1','Argument__2']} pid <5901.334.0> registered_name [] error_info {exit,{{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, {gen_server,call, [{'ns_memcached-default','ns_1@10.88.165.188'}, list_vbuckets_prevstate,30000]}}, [{gen_server,call,3}, {ns_janitor,'-read_existing_map/4-lc$^0/1-0-',2}, {ns_janitor,read_existing_map,4}, {ns_janitor,cleanup,2}, {proc_lib,init_p_do_apply,3}]} ancestors [<5901.304.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<5901.52.0>] messages [] links [<5901.304.0>] dictionary [] trap_exit false status running heap_size 10946 stack_size 24 reductions 275 INFO REPORT <5901.227.0> 2011-07-14 00:18:33 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 1 times in the past 0.332608 secs (last seen 0.332608 secs ago INFO REPORT <5901.304.0> 2011-07-14 00:18:33 =============================================================================== ns_1@10.88.165.188:<5901.304.0>:ns_orchestrator:178: Janitor run exited for bucket "default" with reason {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, [{'ns_memcached-default', 'ns_1@10.88.165.188'}, list_vbuckets_prevstate, 30000]}} PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.375.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.376.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.377.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.378.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.379.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.380.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:18:33 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:18:33 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 22 log lines from memcached INFO REPORT <5901.375.0> 2011-07-14 00:18:33 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.375.0> 2011-07-14 00:18:33 =============================================================================== ** Generic server <5901.375.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.2164>}, init, {1310,617113,235337}, "default",#Port<5901.2451>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.375.0> 2011-07-14 00:18:33 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.375.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.341.0>] messages [] links [<5901.49.0>,<5901.342.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.342.0> 2011-07-14 00:18:33 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.375.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:18:33 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 1 times in the past 0.99204 secs (last seen 0.99204 secs ago PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.384.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.385.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.386.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.387.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.388.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.389.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.384.0> 2011-07-14 00:18:33 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.384.0> 2011-07-14 00:18:33 =============================================================================== ** Generic server <5901.384.0> terminating ** Last message in was list_vbuckets_prevstate ** When Server state == {state,{interval,#Ref<5901.0.0.2239>}, init, {1310,617113,738567}, "default",#Port<5901.2452>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.384.0> 2011-07-14 00:18:33 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.384.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.341.0>] messages [{'$gen_call',{<5901.386.0>,#Ref<5901.0.0.2326>},{stats,<<>>}}] links [<5901.49.0>,<5901.342.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6665 SUPERVISOR REPORT <5901.342.0> 2011-07-14 00:18:33 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.384.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker CRASH REPORT <5901.396.0> 2011-07-14 00:18:33 =============================================================================== Crashing process initial_call {ns_janitor,cleanup,['Argument__1','Argument__2']} pid <5901.396.0> registered_name [] error_info {exit,{{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, {gen_server,call, [{'ns_memcached-default','ns_1@10.88.165.188'}, list_vbuckets_prevstate,30000]}}, [{gen_server,call,3}, {ns_janitor,'-read_existing_map/4-lc$^0/1-0-',2}, {ns_janitor,read_existing_map,4}, {ns_janitor,cleanup,2}, {proc_lib,init_p_do_apply,3}]} ancestors [<5901.304.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<5901.52.0>] messages [] links [<5901.304.0>] dictionary [] trap_exit false status running heap_size 10946 stack_size 24 reductions 224 ERROR REPORT <5901.386.0> 2011-07-14 00:18:33 =============================================================================== ns_1@10.88.165.188:<5901.386.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:18:33 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 2 times in the past 0.998058 secs (last seen 0.66545 secs ago INFO REPORT <5901.304.0> 2011-07-14 00:18:33 =============================================================================== ns_1@10.88.165.188:<5901.304.0>:ns_orchestrator:178: Janitor run exited for bucket "default" with reason {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, [{'ns_memcached-default', 'ns_1@10.88.165.188'}, list_vbuckets_prevstate, 30000]}} PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.399.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.400.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.402.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.403.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.404.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.405.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.399.0> 2011-07-14 00:18:34 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.399.0> 2011-07-14 00:18:34 =============================================================================== ** Generic server <5901.399.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.2336>}, init, {1310,617113,901117}, "default",#Port<5901.2454>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.399.0> 2011-07-14 00:18:34 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.399.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.341.0>] messages [] links [<5901.49.0>,<5901.342.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.342.0> 2011-07-14 00:18:34 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.399.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:18:34 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 2 times in the past 1.656067 secs (last seen 0.664027 secs ago PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.410.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.411.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.412.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.413.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.414.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.415.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:18:34 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:18:34 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.410.0> 2011-07-14 00:18:34 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.410.0> 2011-07-14 00:18:34 =============================================================================== ** Generic server <5901.410.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.2432>}, init, {1310,617114,402631}, "default",#Port<5901.2455>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.410.0> 2011-07-14 00:18:34 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.410.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.341.0>] messages [] links [<5901.49.0>,<5901.342.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.342.0> 2011-07-14 00:18:34 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.410.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.412.0> 2011-07-14 00:18:34 =============================================================================== ns_1@10.88.165.188:<5901.412.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:18:34 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 3 times in the past 1.99737 secs (last seen 0.999312 secs ago PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.419.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] SUPERVISOR REPORT <5901.258.0> 2011-07-14 00:18:34 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.88.165.188'}, {latest,minute,1}]}} pid <5901.263.0> name menelaus_web_alerts_srv mfargs {menelaus_web_alerts_srv,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.421.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.258.0> 2011-07-14 00:18:34 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<5901.422.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.423.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.424.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.425.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.426.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.419.0> 2011-07-14 00:18:35 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.419.0> 2011-07-14 00:18:35 =============================================================================== ** Generic server <5901.419.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.2513>}, init, {1310,617114,900386}, "default",#Port<5901.2456>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.419.0> 2011-07-14 00:18:35 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.419.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.341.0>] messages [] links [<5901.49.0>,<5901.342.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.342.0> 2011-07-14 00:18:35 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.419.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:18:35 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 3 times in the past 2.656263 secs (last seen 1.000196 secs ago PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.430.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.431.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.432.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.433.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.434.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.435.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.430.0> 2011-07-14 00:18:35 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.430.0> 2011-07-14 00:18:35 =============================================================================== ** Generic server <5901.430.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.2600>}, init, {1310,617115,402812}, "default",#Port<5901.2457>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.430.0> 2011-07-14 00:18:35 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.430.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.341.0>] messages [] links [<5901.49.0>,<5901.342.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.342.0> 2011-07-14 00:18:35 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.430.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.432.0> 2011-07-14 00:18:35 =============================================================================== ns_1@10.88.165.188:<5901.432.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:18:35 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 4 times in the past 3.000253 secs (last seen 1.002883 secs ago PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.442.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:18:35 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:18:35 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.443.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.444.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.445.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.446.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.447.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.442.0> 2011-07-14 00:18:36 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.442.0> 2011-07-14 00:18:36 =============================================================================== ** Generic server <5901.442.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.2685>}, init, {1310,617115,903225}, "default",#Port<5901.2458>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.442.0> 2011-07-14 00:18:36 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.442.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.341.0>] messages [] links [<5901.49.0>,<5901.342.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.342.0> 2011-07-14 00:18:36 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.442.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:18:36 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 4 times in the past 3.66024 secs (last seen 1.003977 secs ago PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.452.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.453.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.454.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.455.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.456.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.457.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.452.0> 2011-07-14 00:18:36 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.452.0> 2011-07-14 00:18:36 =============================================================================== ** Generic server <5901.452.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.2793>}, init, {1310,617116,406779}, "default",#Port<5901.2459>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.452.0> 2011-07-14 00:18:36 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.452.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.341.0>] messages [] links [<5901.49.0>,<5901.342.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.342.0> 2011-07-14 00:18:36 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.452.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.454.0> 2011-07-14 00:18:36 =============================================================================== ns_1@10.88.165.188:<5901.454.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:18:36 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 5 times in the past 3.997439 secs (last seen 0.997186 secs ago PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.461.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.462.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.463.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.464.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.465.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.466.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:18:36 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:18:36 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.461.0> 2011-07-14 00:18:37 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.461.0> 2011-07-14 00:18:37 =============================================================================== ** Generic server <5901.461.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.2869>}, init, {1310,617116,900422}, "default",#Port<5901.2460>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.461.0> 2011-07-14 00:18:37 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.461.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.341.0>] messages [] links [<5901.49.0>,<5901.342.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.342.0> 2011-07-14 00:18:37 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.461.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:18:37 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 5 times in the past 4.654974 secs (last seen 0.994734 secs ago PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.470.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.471.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.472.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.473.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.474.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.475.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.470.0> 2011-07-14 00:18:37 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.470.0> 2011-07-14 00:18:37 =============================================================================== ** Generic server <5901.470.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.2960>}, init, {1310,617117,401589}, "default",#Port<5901.2463>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.470.0> 2011-07-14 00:18:37 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.470.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.341.0>] messages [{'$gen_call',{<5901.472.0>,#Ref<5901.0.0.3073>},{stats,<<>>}}] links [<5901.49.0>,<5901.342.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.342.0> 2011-07-14 00:18:37 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.470.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.472.0> 2011-07-14 00:18:37 =============================================================================== ns_1@10.88.165.188:<5901.472.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:18:37 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 6 times in the past 5.156273 secs (last seen 0.501299 secs ago PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.484.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.485.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] SUPERVISOR REPORT <5901.258.0> 2011-07-14 00:18:37 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.88.165.188'}, {latest,minute,1}]}} pid <5901.422.0> name menelaus_web_alerts_srv mfargs {menelaus_web_alerts_srv,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.487.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.258.0> 2011-07-14 00:18:37 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<5901.488.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.489.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.490.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.491.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.484.0> 2011-07-14 00:18:38 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.484.0> 2011-07-14 00:18:38 =============================================================================== ** Generic server <5901.484.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.3083>}, init, {1310,617117,903306}, "default",#Port<5901.2464>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.484.0> 2011-07-14 00:18:38 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.484.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.341.0>] messages [] links [<5901.49.0>,<5901.342.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.342.0> 2011-07-14 00:18:38 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.484.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:18:38 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 7 times in the past 5.657425 secs (last seen 0.501152 secs ago PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.496.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:18:38 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:18:38 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.497.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.498.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.499.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.500.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.501.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.496.0> 2011-07-14 00:18:38 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.496.0> 2011-07-14 00:18:38 =============================================================================== ** Generic server <5901.496.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.3200>}, init, {1310,617118,403920}, "default",#Port<5901.2465>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.496.0> 2011-07-14 00:18:38 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.496.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.341.0>] messages [] links [<5901.49.0>,<5901.342.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6665 SUPERVISOR REPORT <5901.342.0> 2011-07-14 00:18:38 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.496.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.498.0> 2011-07-14 00:18:38 =============================================================================== ns_1@10.88.165.188:<5901.498.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:18:38 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 6 times in the past 5.997314 secs (last seen 1.999875 secs ago PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.508.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.509.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.511.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.512.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.514.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.515.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.508.0> 2011-07-14 00:18:39 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.508.0> 2011-07-14 00:18:39 =============================================================================== ** Generic server <5901.508.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.3298>}, init, {1310,617118,901108}, "default",#Port<5901.2466>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.508.0> 2011-07-14 00:18:39 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.508.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.341.0>] messages [] links [<5901.49.0>,<5901.342.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.342.0> 2011-07-14 00:18:39 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.508.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:18:39 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 8 times in the past 6.656282 secs (last seen 0.998857 secs ago PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.520.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.521.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.522.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.523.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.524.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.525.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:18:39 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:18:39 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.520.0> 2011-07-14 00:18:39 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.520.0> 2011-07-14 00:18:39 =============================================================================== ** Generic server <5901.520.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.3409>}, init, {1310,617119,402659}, "default",#Port<5901.2467>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.520.0> 2011-07-14 00:18:39 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.520.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.341.0>] messages [] links [<5901.49.0>,<5901.342.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.342.0> 2011-07-14 00:18:39 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.520.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.522.0> 2011-07-14 00:18:39 =============================================================================== ns_1@10.88.165.188:<5901.522.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:18:39 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 7 times in the past 6.997375 secs (last seen 1.000061 secs ago PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.532.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.533.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.534.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.535.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.536.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.537.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.532.0> 2011-07-14 00:18:40 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.532.0> 2011-07-14 00:18:40 =============================================================================== ** Generic server <5901.532.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.3495>}, init, {1310,617119,900352}, "default",#Port<5901.2468>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.532.0> 2011-07-14 00:18:40 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.532.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.341.0>] messages [] links [<5901.49.0>,<5901.342.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.342.0> 2011-07-14 00:18:40 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.532.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:18:40 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 9 times in the past 7.656558 secs (last seen 1.000276 secs ago PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.541.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.542.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.543.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.544.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.545.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.546.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.541.0> 2011-07-14 00:18:40 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.541.0> 2011-07-14 00:18:40 =============================================================================== ** Generic server <5901.541.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.3583>}, init, {1310,617120,403227}, "default",#Port<5901.2469>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.541.0> 2011-07-14 00:18:40 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.541.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.341.0>] messages [] links [<5901.49.0>,<5901.342.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.342.0> 2011-07-14 00:18:40 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.541.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.543.0> 2011-07-14 00:18:40 =============================================================================== ns_1@10.88.165.188:<5901.543.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:18:40 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 8 times in the past 7.997363 secs (last seen 0.999988 secs ago PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.551.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.552.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.553.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:18:40 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:18:40 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.554.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.555.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.556.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.551.0> 2011-07-14 00:18:41 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.551.0> 2011-07-14 00:18:41 =============================================================================== ** Generic server <5901.551.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.3679>}, init, {1310,617120,900247}, "default",#Port<5901.2470>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.551.0> 2011-07-14 00:18:41 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.551.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.341.0>] messages [] links [<5901.49.0>,<5901.342.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.342.0> 2011-07-14 00:18:41 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.551.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:18:41 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 10 times in the past 8.656649 secs (last seen 1.000091 secs ago PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.561.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.562.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.563.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.564.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.565.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.342.0> 2011-07-14 00:18:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.566.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.561.0> 2011-07-14 00:18:41 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.561.0> 2011-07-14 00:18:41 =============================================================================== ** Generic server <5901.561.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.3801>}, init, {1310,617121,403243}, "default",#Port<5901.2472>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.561.0> 2011-07-14 00:18:41 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.561.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.341.0>] messages [] links [<5901.49.0>,<5901.342.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.342.0> 2011-07-14 00:18:41 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.561.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker SUPERVISOR REPORT <5901.342.0> 2011-07-14 00:18:41 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext shutdown reason reached_max_restart_intensity pid <5901.561.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.563.0> 2011-07-14 00:18:41 =============================================================================== ns_1@10.88.165.188:<5901.563.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:18:41 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 9 times in the past 8.997482 secs (last seen 1.000119 secs ago INFO REPORT <5901.341.0> 2011-07-14 00:18:41 =============================================================================== ns_1@10.88.165.188:<5901.341.0>:single_bucket_sup:24: per-bucket supervisor for "default" died with reason shutdown SUPERVISOR REPORT <5901.273.0> 2011-07-14 00:18:41 =============================================================================== Reporting supervisor {local,ns_bucket_sup} Child process errorContext child_terminated reason shutdown pid <5901.341.0> name {per_bucket_sup,"default"} mfargs {single_bucket_sup,start_link,["default"]} restart_type permanent shutdown infinity child_type supervisor PROGRESS REPORT <5901.273.0> 2011-07-14 00:18:41 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<5901.573.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.575.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.576.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.577.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.578.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.579.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.580.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.575.0> 2011-07-14 00:18:42 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.575.0> 2011-07-14 00:18:42 =============================================================================== ** Generic server <5901.575.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.3884>}, init, {1310,617121,900767}, "default",#Port<5901.2473>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.575.0> 2011-07-14 00:18:42 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.575.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.573.0>] messages [] links [<5901.49.0>,<5901.574.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.574.0> 2011-07-14 00:18:42 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.575.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:18:42 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 11 times in the past 9.656614 secs (last seen 0.999965 secs ago PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.584.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:18:42 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:18:42 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.585.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.586.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.587.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.588.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.589.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.584.0> 2011-07-14 00:18:42 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.584.0> 2011-07-14 00:18:42 =============================================================================== ** Generic server <5901.584.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.3971>}, init, {1310,617122,403160}, "default",#Port<5901.2474>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.584.0> 2011-07-14 00:18:42 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.584.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.573.0>] messages [] links [<5901.49.0>,<5901.574.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.574.0> 2011-07-14 00:18:42 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.584.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.586.0> 2011-07-14 00:18:42 =============================================================================== ns_1@10.88.165.188:<5901.586.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:18:42 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 10 times in the past 9.997444 secs (last seen 0.999962 secs ago PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.594.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.595.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.596.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.597.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.598.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.599.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.239.0> 2011-07-14 00:18:43 =============================================================================== ns_1@10.88.165.188:<5901.239.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.594.0> 2011-07-14 00:18:43 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.594.0> 2011-07-14 00:18:43 =============================================================================== ** Generic server <5901.594.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.4071>}, init, {1310,617122,900496}, "default",#Port<5901.2475>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.594.0> 2011-07-14 00:18:43 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.594.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.573.0>] messages [] links [<5901.49.0>,<5901.574.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.574.0> 2011-07-14 00:18:43 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.594.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:18:43 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 12 times in the past 10.656499 secs (last seen 0.999885 secs ago PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.605.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.606.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.607.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.608.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.609.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.610.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.605.0> 2011-07-14 00:18:43 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.605.0> 2011-07-14 00:18:43 =============================================================================== ** Generic server <5901.605.0> terminating ** Last message in was list_vbuckets_prevstate ** When Server state == {state,{interval,#Ref<5901.0.0.4164>}, init, {1310,617123,403001}, "default",#Port<5901.2477>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.605.0> 2011-07-14 00:18:43 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.605.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.573.0>] messages [] links [<5901.49.0>,<5901.574.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6671 SUPERVISOR REPORT <5901.574.0> 2011-07-14 00:18:43 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.605.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker CRASH REPORT <5901.620.0> 2011-07-14 00:18:43 =============================================================================== Crashing process initial_call {ns_janitor,cleanup,['Argument__1','Argument__2']} pid <5901.620.0> registered_name [] error_info {exit,{{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, {gen_server,call, [{'ns_memcached-default','ns_1@10.88.165.188'}, list_vbuckets_prevstate,30000]}}, [{gen_server,call,3}, {ns_janitor,'-read_existing_map/4-lc$^0/1-0-',2}, {ns_janitor,read_existing_map,4}, {ns_janitor,cleanup,2}, {proc_lib,init_p_do_apply,3}]} ancestors [<5901.304.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<5901.52.0>] messages [] links [<5901.304.0>] dictionary [] trap_exit false status running heap_size 10946 stack_size 24 reductions 224 INFO REPORT <5901.227.0> 2011-07-14 00:18:43 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 11 times in the past 10.996655 secs (last seen 0.999211 secs ago INFO REPORT <5901.304.0> 2011-07-14 00:18:43 =============================================================================== ns_1@10.88.165.188:<5901.304.0>:ns_orchestrator:178: Janitor run exited for bucket "default" with reason {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, [{'ns_memcached-default', 'ns_1@10.88.165.188'}, list_vbuckets_prevstate, 30000]}} ERROR REPORT <5901.607.0> 2011-07-14 00:18:43 =============================================================================== ns_1@10.88.165.188:<5901.607.0>:stats_collector:94: Exception in stats collector: {exit, {noproc, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.624.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:18:43 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:18:43 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.625.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.627.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.628.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.630.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.631.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.624.0> 2011-07-14 00:18:44 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.624.0> 2011-07-14 00:18:44 =============================================================================== ** Generic server <5901.624.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.4355>}, init, {1310,617123,900817}, "default",#Port<5901.2480>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.624.0> 2011-07-14 00:18:44 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.624.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.573.0>] messages [] links [<5901.49.0>,<5901.574.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.574.0> 2011-07-14 00:18:44 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.624.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:18:44 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 13 times in the past 11.656702 secs (last seen 1.000203 secs ago PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.636.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.637.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.638.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.639.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.640.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.641.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.636.0> 2011-07-14 00:18:44 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.636.0> 2011-07-14 00:18:44 =============================================================================== ** Generic server <5901.636.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.4494>}, init, {1310,617124,403229}, "default",#Port<5901.2482>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.636.0> 2011-07-14 00:18:44 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.636.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.573.0>] messages [] links [<5901.49.0>,<5901.574.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.574.0> 2011-07-14 00:18:44 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.636.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.638.0> 2011-07-14 00:18:44 =============================================================================== ns_1@10.88.165.188:<5901.638.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:18:44 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 12 times in the past 12.000967 secs (last seen 1.004312 secs ago PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.645.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:18:44 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:18:44 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.646.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.647.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.648.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.649.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.650.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.645.0> 2011-07-14 00:18:45 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.645.0> 2011-07-14 00:18:45 =============================================================================== ** Generic server <5901.645.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.4570>}, init, {1310,617124,903957}, "default",#Port<5901.2483>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.645.0> 2011-07-14 00:18:45 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.645.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.573.0>] messages [] links [<5901.49.0>,<5901.574.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.574.0> 2011-07-14 00:18:45 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.645.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:18:45 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 14 times in the past 12.660834 secs (last seen 1.004132 secs ago PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.654.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.655.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.656.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.657.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.658.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.659.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.654.0> 2011-07-14 00:18:45 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.654.0> 2011-07-14 00:18:45 =============================================================================== ** Generic server <5901.654.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.4658>}, init, {1310,617125,407414}, "default",#Port<5901.2484>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.654.0> 2011-07-14 00:18:45 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.654.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.573.0>] messages [] links [<5901.49.0>,<5901.574.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.574.0> 2011-07-14 00:18:45 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.654.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.656.0> 2011-07-14 00:18:45 =============================================================================== ns_1@10.88.165.188:<5901.656.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:18:45 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 13 times in the past 12.997225 secs (last seen 0.996258 secs ago PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.667.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.668.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.669.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.670.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.671.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.672.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:18:45 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:18:45 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.667.0> 2011-07-14 00:18:46 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.667.0> 2011-07-14 00:18:46 =============================================================================== ** Generic server <5901.667.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.4759>}, init, {1310,617125,900163}, "default",#Port<5901.2485>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.667.0> 2011-07-14 00:18:46 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.667.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.573.0>] messages [] links [<5901.49.0>,<5901.574.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.574.0> 2011-07-14 00:18:46 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.667.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:18:46 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 15 times in the past 13.656913 secs (last seen 0.996079 secs ago PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.676.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.677.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.678.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.679.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.680.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.681.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.676.0> 2011-07-14 00:18:46 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.676.0> 2011-07-14 00:18:46 =============================================================================== ** Generic server <5901.676.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.4845>}, init, {1310,617126,403495}, "default",#Port<5901.2486>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.676.0> 2011-07-14 00:18:46 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.676.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.573.0>] messages [] links [<5901.49.0>,<5901.574.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.574.0> 2011-07-14 00:18:46 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.676.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.678.0> 2011-07-14 00:18:46 =============================================================================== ns_1@10.88.165.188:<5901.678.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:18:46 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 14 times in the past 13.997442 secs (last seen 1.000217 secs ago PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.685.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.686.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.687.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.688.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.689.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.690.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.685.0> 2011-07-14 00:18:47 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.685.0> 2011-07-14 00:18:47 =============================================================================== ** Generic server <5901.685.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.4928>}, init, {1310,617126,900557}, "default",#Port<5901.2487>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.685.0> 2011-07-14 00:18:47 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.685.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.573.0>] messages [] links [<5901.49.0>,<5901.574.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.574.0> 2011-07-14 00:18:47 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.685.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:18:47 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 16 times in the past 14.656946 secs (last seen 1.000033 secs ago PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.695.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.696.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.697.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:18:47 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:18:47 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.698.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.699.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.700.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.695.0> 2011-07-14 00:18:47 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.695.0> 2011-07-14 00:18:47 =============================================================================== ** Generic server <5901.695.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.5046>}, init, {1310,617127,403489}, "default",#Port<5901.2489>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.695.0> 2011-07-14 00:18:47 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.695.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.573.0>] messages [] links [<5901.49.0>,<5901.574.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.574.0> 2011-07-14 00:18:47 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.695.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.697.0> 2011-07-14 00:18:47 =============================================================================== ns_1@10.88.165.188:<5901.697.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:18:47 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 15 times in the past 14.997314 secs (last seen 0.999872 secs ago PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.707.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.708.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.709.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.710.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.711.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.712.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.707.0> 2011-07-14 00:18:48 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.707.0> 2011-07-14 00:18:48 =============================================================================== ** Generic server <5901.707.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.5131>}, init, {1310,617127,900368}, "default",#Port<5901.2490>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.707.0> 2011-07-14 00:18:48 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.707.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.573.0>] messages [] links [<5901.49.0>,<5901.574.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.574.0> 2011-07-14 00:18:48 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.707.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:18:48 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 17 times in the past 15.6569 secs (last seen 0.999954 secs ago PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.716.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.717.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.718.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.719.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.720.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.721.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.716.0> 2011-07-14 00:18:48 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.716.0> 2011-07-14 00:18:48 =============================================================================== ** Generic server <5901.716.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.5219>}, init, {1310,617128,403390}, "default",#Port<5901.2491>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.716.0> 2011-07-14 00:18:48 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.716.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.573.0>] messages [] links [<5901.49.0>,<5901.574.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6665 SUPERVISOR REPORT <5901.574.0> 2011-07-14 00:18:48 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.716.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.718.0> 2011-07-14 00:18:48 =============================================================================== ns_1@10.88.165.188:<5901.718.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:18:48 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 16 times in the past 15.998084 secs (last seen 1.00077 secs ago PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.729.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:18:48 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:18:48 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.730.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.732.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.733.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.735.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.736.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.729.0> 2011-07-14 00:18:49 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.729.0> 2011-07-14 00:18:49 =============================================================================== ** Generic server <5901.729.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.5331>}, init, {1310,617128,901009}, "default",#Port<5901.2492>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.729.0> 2011-07-14 00:18:49 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.729.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.573.0>] messages [] links [<5901.49.0>,<5901.574.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.574.0> 2011-07-14 00:18:49 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.729.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:18:49 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 18 times in the past 16.656948 secs (last seen 1.000048 secs ago PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.740.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.741.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.742.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.743.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.744.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.745.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.740.0> 2011-07-14 00:18:49 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.740.0> 2011-07-14 00:18:49 =============================================================================== ** Generic server <5901.740.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.5429>}, init, {1310,617129,403473}, "default",#Port<5901.2493>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.740.0> 2011-07-14 00:18:49 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.740.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.573.0>] messages [] links [<5901.49.0>,<5901.574.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.574.0> 2011-07-14 00:18:49 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.740.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.742.0> 2011-07-14 00:18:49 =============================================================================== ns_1@10.88.165.188:<5901.742.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:18:49 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 17 times in the past 16.998205 secs (last seen 1.000121 secs ago PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.752.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.753.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.754.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:18:49 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:18:49 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.755.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.756.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.757.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.752.0> 2011-07-14 00:18:50 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.752.0> 2011-07-14 00:18:50 =============================================================================== ** Generic server <5901.752.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.5511>}, init, {1310,617129,901288}, "default",#Port<5901.2496>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.752.0> 2011-07-14 00:18:50 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.752.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.573.0>] messages [] links [<5901.49.0>,<5901.574.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.574.0> 2011-07-14 00:18:50 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.752.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:18:50 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 19 times in the past 17.657134 secs (last seen 1.000186 secs ago PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.762.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.763.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.764.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.765.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.766.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.767.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.762.0> 2011-07-14 00:18:50 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.762.0> 2011-07-14 00:18:50 =============================================================================== ** Generic server <5901.762.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.5641>}, init, {1310,617130,403705}, "default",#Port<5901.2498>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.762.0> 2011-07-14 00:18:50 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.762.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.573.0>] messages [] links [<5901.49.0>,<5901.574.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.574.0> 2011-07-14 00:18:50 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.762.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.764.0> 2011-07-14 00:18:50 =============================================================================== ns_1@10.88.165.188:<5901.764.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:18:50 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 18 times in the past 17.997453 secs (last seen 0.999248 secs ago PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.771.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.772.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.773.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.774.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.775.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.776.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.771.0> 2011-07-14 00:18:51 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.771.0> 2011-07-14 00:18:51 =============================================================================== ** Generic server <5901.771.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.5723>}, init, {1310,617130,901435}, "default",#Port<5901.2499>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.771.0> 2011-07-14 00:18:51 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.771.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.573.0>] messages [] links [<5901.49.0>,<5901.574.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.574.0> 2011-07-14 00:18:51 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.771.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:18:51 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 20 times in the past 18.657183 secs (last seen 1.000049 secs ago PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.780.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.781.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.782.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:18:51 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:18:51 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.783.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.784.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.785.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.780.0> 2011-07-14 00:18:51 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.780.0> 2011-07-14 00:18:51 =============================================================================== ** Generic server <5901.780.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.5805>}, init, {1310,617131,403775}, "default",#Port<5901.2500>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.780.0> 2011-07-14 00:18:51 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.780.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.573.0>] messages [] links [<5901.49.0>,<5901.574.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.574.0> 2011-07-14 00:18:51 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.780.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.782.0> 2011-07-14 00:18:51 =============================================================================== ns_1@10.88.165.188:<5901.782.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:18:51 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 19 times in the past 18.99814 secs (last seen 1.000687 secs ago PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.792.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.793.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.794.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.795.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.796.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.574.0> 2011-07-14 00:18:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.797.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.792.0> 2011-07-14 00:18:52 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.792.0> 2011-07-14 00:18:52 =============================================================================== ** Generic server <5901.792.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.5891>}, init, {1310,617131,901120}, "default",#Port<5901.2501>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.792.0> 2011-07-14 00:18:52 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.792.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.573.0>] messages [] links [<5901.49.0>,<5901.574.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.574.0> 2011-07-14 00:18:52 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.792.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker SUPERVISOR REPORT <5901.574.0> 2011-07-14 00:18:52 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext shutdown reason reached_max_restart_intensity pid <5901.792.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:18:52 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 21 times in the past 19.657202 secs (last seen 1.000019 secs ago INFO REPORT <5901.573.0> 2011-07-14 00:18:52 =============================================================================== ns_1@10.88.165.188:<5901.573.0>:single_bucket_sup:24: per-bucket supervisor for "default" died with reason shutdown SUPERVISOR REPORT <5901.273.0> 2011-07-14 00:18:52 =============================================================================== Reporting supervisor {local,ns_bucket_sup} Child process errorContext child_terminated reason shutdown pid <5901.573.0> name {per_bucket_sup,"default"} mfargs {single_bucket_sup,start_link,["default"]} restart_type permanent shutdown infinity child_type supervisor PROGRESS REPORT <5901.273.0> 2011-07-14 00:18:52 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<5901.802.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.804.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.805.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.806.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.807.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.808.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.809.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.804.0> 2011-07-14 00:18:52 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.804.0> 2011-07-14 00:18:52 =============================================================================== ** Generic server <5901.804.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.5999>}, init, {1310,617132,404072}, "default",#Port<5901.2502>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.804.0> 2011-07-14 00:18:52 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.804.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.802.0>] messages [] links [<5901.49.0>,<5901.803.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.803.0> 2011-07-14 00:18:52 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.804.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.806.0> 2011-07-14 00:18:52 =============================================================================== ns_1@10.88.165.188:<5901.806.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:18:52 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 20 times in the past 19.99739 secs (last seen 0.99925 secs ago PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.814.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.815.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.816.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:18:52 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:18:52 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.817.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.818.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.819.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.814.0> 2011-07-14 00:18:53 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.814.0> 2011-07-14 00:18:53 =============================================================================== ** Generic server <5901.814.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.6095>}, init, {1310,617132,900327}, "default",#Port<5901.2503>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.814.0> 2011-07-14 00:18:53 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.814.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.802.0>] messages [] links [<5901.49.0>,<5901.803.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.803.0> 2011-07-14 00:18:53 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.814.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:18:53 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 22 times in the past 20.657285 secs (last seen 1.000083 secs ago PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.824.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.825.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.826.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.827.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.828.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.829.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.824.0> 2011-07-14 00:18:53 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.824.0> 2011-07-14 00:18:53 =============================================================================== ** Generic server <5901.824.0> terminating ** Last message in was list_vbuckets_prevstate ** When Server state == {state,{interval,#Ref<5901.0.0.6218>}, init, {1310,617133,403907}, "default",#Port<5901.2506>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.824.0> 2011-07-14 00:18:53 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.824.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.802.0>] messages [] links [<5901.49.0>,<5901.803.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6671 SUPERVISOR REPORT <5901.803.0> 2011-07-14 00:18:53 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.824.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker CRASH REPORT <5901.836.0> 2011-07-14 00:18:53 =============================================================================== Crashing process initial_call {ns_janitor,cleanup,['Argument__1','Argument__2']} pid <5901.836.0> registered_name [] error_info {exit,{{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, {gen_server,call, [{'ns_memcached-default','ns_1@10.88.165.188'}, list_vbuckets_prevstate,30000]}}, [{gen_server,call,3}, {ns_janitor,'-read_existing_map/4-lc$^0/1-0-',2}, {ns_janitor,read_existing_map,4}, {ns_janitor,cleanup,2}, {proc_lib,init_p_do_apply,3}]} ancestors [<5901.304.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<5901.52.0>] messages [] links [<5901.304.0>] dictionary [] trap_exit false status running heap_size 10946 stack_size 24 reductions 224 INFO REPORT <5901.227.0> 2011-07-14 00:18:53 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 21 times in the past 20.995841 secs (last seen 0.998451 secs ago INFO REPORT <5901.304.0> 2011-07-14 00:18:53 =============================================================================== ns_1@10.88.165.188:<5901.304.0>:ns_orchestrator:178: Janitor run exited for bucket "default" with reason {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, [{'ns_memcached-default', 'ns_1@10.88.165.188'}, list_vbuckets_prevstate, 30000]}} ERROR REPORT <5901.826.0> 2011-07-14 00:18:53 =============================================================================== ns_1@10.88.165.188:<5901.826.0>:stats_collector:94: Exception in stats collector: {exit, {noproc, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.842.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.843.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.845.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.846.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.847.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.848.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.842.0> 2011-07-14 00:18:54 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.842.0> 2011-07-14 00:18:54 =============================================================================== ** Generic server <5901.842.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.6330>}, init, {1310,617133,899531}, "default",#Port<5901.2508>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.842.0> 2011-07-14 00:18:54 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.842.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.802.0>] messages [] links [<5901.49.0>,<5901.803.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.803.0> 2011-07-14 00:18:54 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.842.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:18:54 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 23 times in the past 21.654169 secs (last seen 0.996884 secs ago PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.853.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.854.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.855.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.856.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.857.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.858.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:18:54 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:18:54 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.853.0> 2011-07-14 00:18:54 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.853.0> 2011-07-14 00:18:54 =============================================================================== ** Generic server <5901.853.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.6427>}, init, {1310,617134,400705}, "default",#Port<5901.2509>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.853.0> 2011-07-14 00:18:54 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.853.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.802.0>] messages [] links [<5901.49.0>,<5901.803.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.803.0> 2011-07-14 00:18:54 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.853.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.855.0> 2011-07-14 00:18:54 =============================================================================== ns_1@10.88.165.188:<5901.855.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:18:54 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 22 times in the past 21.997432 secs (last seen 1.001591 secs ago PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.862.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.863.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.864.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.865.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.866.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.867.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.862.0> 2011-07-14 00:18:55 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.862.0> 2011-07-14 00:18:55 =============================================================================== ** Generic server <5901.862.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.6507>}, init, {1310,617134,900434}, "default",#Port<5901.2510>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.862.0> 2011-07-14 00:18:55 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.862.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.802.0>] messages [] links [<5901.49.0>,<5901.803.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.803.0> 2011-07-14 00:18:55 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.862.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:18:55 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 24 times in the past 22.65454 secs (last seen 1.000371 secs ago PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.872.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.873.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.874.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.875.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.876.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.877.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.872.0> 2011-07-14 00:18:55 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.872.0> 2011-07-14 00:18:55 =============================================================================== ** Generic server <5901.872.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.6611>}, init, {1310,617135,401090}, "default",#Port<5901.2511>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.872.0> 2011-07-14 00:18:55 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.872.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.802.0>] messages [] links [<5901.49.0>,<5901.803.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.803.0> 2011-07-14 00:18:55 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.872.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.874.0> 2011-07-14 00:18:55 =============================================================================== ns_1@10.88.165.188:<5901.874.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:18:55 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 23 times in the past 22.99751 secs (last seen 1.000078 secs ago PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.884.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.885.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:18:55 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:18:55 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.886.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.887.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.888.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.889.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.884.0> 2011-07-14 00:18:56 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.884.0> 2011-07-14 00:18:56 =============================================================================== ** Generic server <5901.884.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.6696>}, init, {1310,617135,900887}, "default",#Port<5901.2512>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.884.0> 2011-07-14 00:18:56 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.884.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.802.0>] messages [] links [<5901.49.0>,<5901.803.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.803.0> 2011-07-14 00:18:56 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.884.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:18:56 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 25 times in the past 23.654643 secs (last seen 1.000103 secs ago PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.894.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.895.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.896.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.897.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.898.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.899.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.894.0> 2011-07-14 00:18:56 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.894.0> 2011-07-14 00:18:56 =============================================================================== ** Generic server <5901.894.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.6823>}, init, {1310,617136,401189}, "default",#Port<5901.2514>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.894.0> 2011-07-14 00:18:56 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.894.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.802.0>] messages [] links [<5901.49.0>,<5901.803.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.803.0> 2011-07-14 00:18:56 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.894.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.896.0> 2011-07-14 00:18:56 =============================================================================== ns_1@10.88.165.188:<5901.896.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:18:56 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 24 times in the past 23.997598 secs (last seen 1.000088 secs ago PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.903.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.904.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.905.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.906.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.907.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.908.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.903.0> 2011-07-14 00:18:57 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.903.0> 2011-07-14 00:18:57 =============================================================================== ** Generic server <5901.903.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.6901>}, init, {1310,617136,900715}, "default",#Port<5901.2515>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.903.0> 2011-07-14 00:18:57 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.903.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.802.0>] messages [] links [<5901.49.0>,<5901.803.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.803.0> 2011-07-14 00:18:57 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.903.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:18:57 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 26 times in the past 24.654698 secs (last seen 1.000055 secs ago PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.912.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:18:57 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:18:57 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.913.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.914.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.915.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.916.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.917.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.912.0> 2011-07-14 00:18:57 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.912.0> 2011-07-14 00:18:57 =============================================================================== ** Generic server <5901.912.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.6987>}, init, {1310,617137,401119}, "default",#Port<5901.2516>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.912.0> 2011-07-14 00:18:57 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.912.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.802.0>] messages [] links [<5901.49.0>,<5901.803.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.803.0> 2011-07-14 00:18:57 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.912.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.914.0> 2011-07-14 00:18:57 =============================================================================== ns_1@10.88.165.188:<5901.914.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:18:57 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 25 times in the past 24.997704 secs (last seen 1.000106 secs ago PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.924.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.925.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.926.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.927.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.928.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.929.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.924.0> 2011-07-14 00:18:58 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.924.0> 2011-07-14 00:18:58 =============================================================================== ** Generic server <5901.924.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.7073>}, init, {1310,617137,901498}, "default",#Port<5901.2517>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.924.0> 2011-07-14 00:18:58 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.924.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.802.0>] messages [] links [<5901.49.0>,<5901.803.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.803.0> 2011-07-14 00:18:58 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.924.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:18:58 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 27 times in the past 25.657479 secs (last seen 1.002781 secs ago PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.935.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.936.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.937.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.938.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.939.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.940.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.935.0> 2011-07-14 00:18:58 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.935.0> 2011-07-14 00:18:58 =============================================================================== ** Generic server <5901.935.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.7202>}, init, {1310,617138,403977}, "default",#Port<5901.2518>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.935.0> 2011-07-14 00:18:58 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.935.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.802.0>] messages [] links [<5901.49.0>,<5901.803.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6665 SUPERVISOR REPORT <5901.803.0> 2011-07-14 00:18:58 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.935.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.937.0> 2011-07-14 00:18:58 =============================================================================== ns_1@10.88.165.188:<5901.937.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:18:58 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 26 times in the past 25.99784 secs (last seen 1.000136 secs ago PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.948.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:18:58 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:18:58 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.949.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.951.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.952.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.954.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.955.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.948.0> 2011-07-14 00:18:59 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.948.0> 2011-07-14 00:18:59 =============================================================================== ** Generic server <5901.948.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.7375>}, init, {1310,617138,900780}, "default",#Port<5901.2520>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.948.0> 2011-07-14 00:18:59 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.948.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.802.0>] messages [] links [<5901.49.0>,<5901.803.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.803.0> 2011-07-14 00:18:59 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.948.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:18:59 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 28 times in the past 26.657697 secs (last seen 1.000218 secs ago PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.960.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.961.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.962.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.963.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.964.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.965.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.960.0> 2011-07-14 00:18:59 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.960.0> 2011-07-14 00:18:59 =============================================================================== ** Generic server <5901.960.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.7502>}, init, {1310,617139,404296}, "default",#Port<5901.2522>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.960.0> 2011-07-14 00:18:59 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.960.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.802.0>] messages [] links [<5901.49.0>,<5901.803.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.803.0> 2011-07-14 00:18:59 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.960.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.962.0> 2011-07-14 00:18:59 =============================================================================== ns_1@10.88.165.188:<5901.962.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:18:59 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 27 times in the past 26.997764 secs (last seen 0.999924 secs ago PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.972.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.973.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.974.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.975.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.976.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:18:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.977.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.972.0> 2011-07-14 00:19:00 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.972.0> 2011-07-14 00:19:00 =============================================================================== ** Generic server <5901.972.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.7589>}, init, {1310,617139,901678}, "default",#Port<5901.2523>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.972.0> 2011-07-14 00:19:00 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.972.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.802.0>] messages [] links [<5901.49.0>,<5901.803.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.803.0> 2011-07-14 00:19:00 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.972.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:00 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 29 times in the past 27.654938 secs (last seen 0.997241 secs ago PROGRESS REPORT <5901.803.0> 2011-07-14 00:19:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.981.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:19:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.982.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:19:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.983.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:19:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.984.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:19:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.985.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:19:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.986.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:00 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:00 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.981.0> 2011-07-14 00:19:00 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.981.0> 2011-07-14 00:19:00 =============================================================================== ** Generic server <5901.981.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.7676>}, init, {1310,617140,401411}, "default",#Port<5901.2524>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.981.0> 2011-07-14 00:19:00 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.981.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.802.0>] messages [] links [<5901.49.0>,<5901.803.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.803.0> 2011-07-14 00:19:00 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.981.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.983.0> 2011-07-14 00:19:00 =============================================================================== ns_1@10.88.165.188:<5901.983.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:00 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 28 times in the past 27.998293 secs (last seen 1.000529 secs ago PROGRESS REPORT <5901.803.0> 2011-07-14 00:19:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.990.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:19:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.991.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:19:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.992.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:19:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.993.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:19:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.994.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:19:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.995.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.990.0> 2011-07-14 00:19:01 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.990.0> 2011-07-14 00:19:01 =============================================================================== ** Generic server <5901.990.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.7752>}, init, {1310,617140,901374}, "default",#Port<5901.2525>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.990.0> 2011-07-14 00:19:01 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.990.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.802.0>] messages [] links [<5901.49.0>,<5901.803.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.803.0> 2011-07-14 00:19:01 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.990.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:01 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 30 times in the past 28.657896 secs (last seen 1.002958 secs ago PROGRESS REPORT <5901.803.0> 2011-07-14 00:19:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1000.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:19:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1001.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:19:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1002.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:19:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1003.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:19:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1004.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:19:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1005.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1000.0> 2011-07-14 00:19:01 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1000.0> 2011-07-14 00:19:01 =============================================================================== ** Generic server <5901.1000.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.7860>}, init, {1310,617141,404439}, "default",#Port<5901.2526>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1000.0> 2011-07-14 00:19:01 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1000.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.802.0>] messages [] links [<5901.49.0>,<5901.803.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.803.0> 2011-07-14 00:19:01 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1000.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1002.0> 2011-07-14 00:19:01 =============================================================================== ns_1@10.88.165.188:<5901.1002.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:01 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 29 times in the past 28.998006 secs (last seen 0.999713 secs ago PROGRESS REPORT <5901.803.0> 2011-07-14 00:19:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1012.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:19:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1013.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:19:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1014.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:01 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:01 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.803.0> 2011-07-14 00:19:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1015.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:19:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1016.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:19:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1017.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1012.0> 2011-07-14 00:19:02 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1012.0> 2011-07-14 00:19:02 =============================================================================== ** Generic server <5901.1012.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.7941>}, init, {1310,617141,900943}, "default",#Port<5901.2527>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1012.0> 2011-07-14 00:19:02 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1012.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.802.0>] messages [] links [<5901.49.0>,<5901.803.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.803.0> 2011-07-14 00:19:02 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1012.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:02 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 31 times in the past 29.657927 secs (last seen 1.000031 secs ago PROGRESS REPORT <5901.803.0> 2011-07-14 00:19:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1022.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:19:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1023.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:19:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1024.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:19:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1025.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:19:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1026.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.803.0> 2011-07-14 00:19:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1027.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1022.0> 2011-07-14 00:19:02 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1022.0> 2011-07-14 00:19:02 =============================================================================== ** Generic server <5901.1022.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.8071>}, init, {1310,617142,405818}, "default",#Port<5901.2529>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1022.0> 2011-07-14 00:19:02 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1022.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.802.0>] messages [] links [<5901.49.0>,<5901.803.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.803.0> 2011-07-14 00:19:02 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1022.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker SUPERVISOR REPORT <5901.803.0> 2011-07-14 00:19:02 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext shutdown reason reached_max_restart_intensity pid <5901.1022.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1024.0> 2011-07-14 00:19:02 =============================================================================== ns_1@10.88.165.188:<5901.1024.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:02 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 30 times in the past 29.999324 secs (last seen 1.001318 secs ago INFO REPORT <5901.802.0> 2011-07-14 00:19:02 =============================================================================== ns_1@10.88.165.188:<5901.802.0>:single_bucket_sup:24: per-bucket supervisor for "default" died with reason shutdown SUPERVISOR REPORT <5901.273.0> 2011-07-14 00:19:02 =============================================================================== Reporting supervisor {local,ns_bucket_sup} Child process errorContext child_terminated reason shutdown pid <5901.802.0> name {per_bucket_sup,"default"} mfargs {single_bucket_sup,start_link,["default"]} restart_type permanent shutdown infinity child_type supervisor PROGRESS REPORT <5901.273.0> 2011-07-14 00:19:02 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<5901.1031.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1033.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1034.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1035.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1036.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1037.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1038.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1033.0> 2011-07-14 00:19:03 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1033.0> 2011-07-14 00:19:03 =============================================================================== ** Generic server <5901.1033.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.8154>}, init, {1310,617142,903012}, "default",#Port<5901.2530>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1033.0> 2011-07-14 00:19:03 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1033.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1031.0>] messages [] links [<5901.49.0>,<5901.1032.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1032.0> 2011-07-14 00:19:03 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1033.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:03 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 32 times in the past 30.657969 secs (last seen 1.000042 secs ago PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1042.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1043.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1044.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:03 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:03 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1045.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1046.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1047.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1042.0> 2011-07-14 00:19:03 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1042.0> 2011-07-14 00:19:03 =============================================================================== ** Generic server <5901.1042.0> terminating ** Last message in was list_vbuckets_prevstate ** When Server state == {state,{interval,#Ref<5901.0.0.8236>}, init, {1310,617143,404598}, "default",#Port<5901.2531>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1042.0> 2011-07-14 00:19:03 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1042.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1031.0>] messages [{'$gen_call',{<5901.1044.0>,#Ref<5901.0.0.8341>},{stats,<<>>}}] links [<5901.49.0>,<5901.1032.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6671 SUPERVISOR REPORT <5901.1032.0> 2011-07-14 00:19:03 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1042.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker CRASH REPORT <5901.1054.0> 2011-07-14 00:19:03 =============================================================================== Crashing process initial_call {ns_janitor,cleanup,['Argument__1','Argument__2']} pid <5901.1054.0> registered_name [] error_info {exit,{{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, {gen_server,call, [{'ns_memcached-default','ns_1@10.88.165.188'}, list_vbuckets_prevstate,30000]}}, [{gen_server,call,3}, {ns_janitor,'-read_existing_map/4-lc$^0/1-0-',2}, {ns_janitor,read_existing_map,4}, {ns_janitor,cleanup,2}, {proc_lib,init_p_do_apply,3}]} ancestors [<5901.304.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<5901.52.0>] messages [] links [<5901.304.0>] dictionary [] trap_exit false status running heap_size 10946 stack_size 24 reductions 224 ERROR REPORT <5901.1044.0> 2011-07-14 00:19:03 =============================================================================== ns_1@10.88.165.188:<5901.1044.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:03 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 31 times in the past 30.997169 secs (last seen 0.997845 secs ago INFO REPORT <5901.304.0> 2011-07-14 00:19:03 =============================================================================== ns_1@10.88.165.188:<5901.304.0>:ns_orchestrator:178: Janitor run exited for bucket "default" with reason {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, [{'ns_memcached-default', 'ns_1@10.88.165.188'}, list_vbuckets_prevstate, 30000]}} PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1060.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1061.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1063.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1064.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1065.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1066.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1060.0> 2011-07-14 00:19:04 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1060.0> 2011-07-14 00:19:04 =============================================================================== ** Generic server <5901.1060.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.8351>}, init, {1310,617143,899993}, "default",#Port<5901.2533>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1060.0> 2011-07-14 00:19:04 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1060.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1031.0>] messages [] links [<5901.49.0>,<5901.1032.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.1032.0> 2011-07-14 00:19:04 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1060.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:04 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 33 times in the past 31.654288 secs (last seen 0.996319 secs ago PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1071.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1072.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1073.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1074.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1075.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1076.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1071.0> 2011-07-14 00:19:04 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1071.0> 2011-07-14 00:19:04 =============================================================================== ** Generic server <5901.1071.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.8449>}, init, {1310,617144,400895}, "default",#Port<5901.2534>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1071.0> 2011-07-14 00:19:04 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1071.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1031.0>] messages [] links [<5901.49.0>,<5901.1032.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1032.0> 2011-07-14 00:19:04 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1071.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1073.0> 2011-07-14 00:19:04 =============================================================================== ns_1@10.88.165.188:<5901.1073.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:04 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 32 times in the past 31.998272 secs (last seen 1.001103 secs ago PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1081.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:04 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:04 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1082.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1083.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1084.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1085.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1086.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1081.0> 2011-07-14 00:19:05 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1081.0> 2011-07-14 00:19:05 =============================================================================== ** Generic server <5901.1081.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.8545>}, init, {1310,617144,901340}, "default",#Port<5901.2535>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1081.0> 2011-07-14 00:19:05 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1081.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1031.0>] messages [] links [<5901.49.0>,<5901.1032.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1032.0> 2011-07-14 00:19:05 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1081.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:05 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 34 times in the past 32.655455 secs (last seen 1.001167 secs ago PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1091.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1092.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1093.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1094.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1095.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1096.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1091.0> 2011-07-14 00:19:05 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1091.0> 2011-07-14 00:19:05 =============================================================================== ** Generic server <5901.1091.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.8670>}, init, {1310,617145,402043}, "default",#Port<5901.2537>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1091.0> 2011-07-14 00:19:05 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1091.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1031.0>] messages [] links [<5901.49.0>,<5901.1032.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1032.0> 2011-07-14 00:19:05 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1091.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1093.0> 2011-07-14 00:19:05 =============================================================================== ns_1@10.88.165.188:<5901.1093.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:05 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 33 times in the past 32.998147 secs (last seen 0.999875 secs ago PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1103.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1104.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1105.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:05 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:05 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1106.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1107.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1108.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1103.0> 2011-07-14 00:19:06 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1103.0> 2011-07-14 00:19:06 =============================================================================== ** Generic server <5901.1103.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.8751>}, init, {1310,617145,901275}, "default",#Port<5901.2538>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1103.0> 2011-07-14 00:19:06 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1103.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1031.0>] messages [] links [<5901.49.0>,<5901.1032.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1032.0> 2011-07-14 00:19:06 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1103.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:06 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 35 times in the past 33.655357 secs (last seen 0.999902 secs ago PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1112.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1113.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1114.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1115.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1116.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1117.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1112.0> 2011-07-14 00:19:06 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1112.0> 2011-07-14 00:19:06 =============================================================================== ** Generic server <5901.1112.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.8837>}, init, {1310,617146,401870}, "default",#Port<5901.2539>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1112.0> 2011-07-14 00:19:06 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1112.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1031.0>] messages [] links [<5901.49.0>,<5901.1032.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1032.0> 2011-07-14 00:19:06 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1112.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1114.0> 2011-07-14 00:19:06 =============================================================================== ns_1@10.88.165.188:<5901.1114.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:06 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 34 times in the past 33.998339 secs (last seen 1.000192 secs ago PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1121.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1122.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1123.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1124.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1125.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1126.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1121.0> 2011-07-14 00:19:07 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1121.0> 2011-07-14 00:19:07 =============================================================================== ** Generic server <5901.1121.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.8919>}, init, {1310,617146,901944}, "default",#Port<5901.2540>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1121.0> 2011-07-14 00:19:07 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1121.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1031.0>] messages [] links [<5901.49.0>,<5901.1032.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1032.0> 2011-07-14 00:19:07 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1121.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:07 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 36 times in the past 34.655561 secs (last seen 1.000204 secs ago PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1130.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:07 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:07 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1131.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1132.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1133.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1134.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1135.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1130.0> 2011-07-14 00:19:07 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1130.0> 2011-07-14 00:19:07 =============================================================================== ** Generic server <5901.1130.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.9001>}, init, {1310,617147,402083}, "default",#Port<5901.2541>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1130.0> 2011-07-14 00:19:07 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1130.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1031.0>] messages [] links [<5901.49.0>,<5901.1032.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1032.0> 2011-07-14 00:19:07 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1130.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1132.0> 2011-07-14 00:19:07 =============================================================================== ns_1@10.88.165.188:<5901.1132.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:07 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 35 times in the past 34.998289 secs (last seen 0.99995 secs ago PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1143.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1144.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1145.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1146.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1147.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1148.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1143.0> 2011-07-14 00:19:08 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1143.0> 2011-07-14 00:19:08 =============================================================================== ** Generic server <5901.1143.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.9109>}, init, {1310,617147,901461}, "default",#Port<5901.2542>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1143.0> 2011-07-14 00:19:08 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1143.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1031.0>] messages [] links [<5901.49.0>,<5901.1032.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1032.0> 2011-07-14 00:19:08 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1143.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:08 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 37 times in the past 35.655483 secs (last seen 0.999922 secs ago PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1153.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:08 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:08 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1154.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1155.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1156.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1157.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1158.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1153.0> 2011-07-14 00:19:08 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1153.0> 2011-07-14 00:19:08 =============================================================================== ** Generic server <5901.1153.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.9235>}, init, {1310,617148,402017}, "default",#Port<5901.2544>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1153.0> 2011-07-14 00:19:08 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1153.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1031.0>] messages [] links [<5901.49.0>,<5901.1032.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6665 SUPERVISOR REPORT <5901.1032.0> 2011-07-14 00:19:08 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1153.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1155.0> 2011-07-14 00:19:08 =============================================================================== ns_1@10.88.165.188:<5901.1155.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:08 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 36 times in the past 35.998324 secs (last seen 1.000035 secs ago PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1165.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1166.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1168.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1169.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1170.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1171.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1165.0> 2011-07-14 00:19:09 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1165.0> 2011-07-14 00:19:09 =============================================================================== ** Generic server <5901.1165.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.9327>}, init, {1310,617148,901102}, "default",#Port<5901.2545>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1165.0> 2011-07-14 00:19:09 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1165.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1031.0>] messages [] links [<5901.49.0>,<5901.1032.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.1032.0> 2011-07-14 00:19:09 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1165.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:09 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 38 times in the past 36.655513 secs (last seen 1.00003 secs ago PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1176.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1177.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1178.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1179.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1180.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1181.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1176.0> 2011-07-14 00:19:09 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1176.0> 2011-07-14 00:19:09 =============================================================================== ** Generic server <5901.1176.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.9430>}, init, {1310,617149,402062}, "default",#Port<5901.2546>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1176.0> 2011-07-14 00:19:09 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1176.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1031.0>] messages [] links [<5901.49.0>,<5901.1032.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1032.0> 2011-07-14 00:19:09 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1176.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1178.0> 2011-07-14 00:19:09 =============================================================================== ns_1@10.88.165.188:<5901.1178.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:09 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 37 times in the past 36.99846 secs (last seen 1.000136 secs ago PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1188.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:09 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:09 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1189.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1190.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1191.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1192.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1193.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1188.0> 2011-07-14 00:19:10 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1188.0> 2011-07-14 00:19:10 =============================================================================== ** Generic server <5901.1188.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.9511>}, init, {1310,617149,901479}, "default",#Port<5901.2547>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1188.0> 2011-07-14 00:19:10 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1188.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1031.0>] messages [] links [<5901.49.0>,<5901.1032.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1032.0> 2011-07-14 00:19:10 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1188.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:10 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 39 times in the past 37.655667 secs (last seen 1.000154 secs ago PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1197.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1198.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1199.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1200.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1201.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1202.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1197.0> 2011-07-14 00:19:10 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1197.0> 2011-07-14 00:19:10 =============================================================================== ** Generic server <5901.1197.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.9600>}, init, {1310,617150,402281}, "default",#Port<5901.2548>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1197.0> 2011-07-14 00:19:10 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1197.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1031.0>] messages [] links [<5901.49.0>,<5901.1032.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1032.0> 2011-07-14 00:19:10 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1197.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1199.0> 2011-07-14 00:19:10 =============================================================================== ns_1@10.88.165.188:<5901.1199.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:10 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 38 times in the past 37.99861 secs (last seen 1.00015 secs ago PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1207.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:10 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:10 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1208.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] SUPERVISOR REPORT <5901.258.0> 2011-07-14 00:19:10 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.88.165.188'}, {latest,minute,1}]}} pid <5901.488.0> name menelaus_web_alerts_srv mfargs {menelaus_web_alerts_srv,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1210.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.258.0> 2011-07-14 00:19:10 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<5901.1211.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1212.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1213.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1214.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1207.0> 2011-07-14 00:19:11 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1207.0> 2011-07-14 00:19:11 =============================================================================== ** Generic server <5901.1207.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.9700>}, init, {1310,617150,903668}, "default",#Port<5901.2549>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1207.0> 2011-07-14 00:19:11 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1207.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1031.0>] messages [] links [<5901.49.0>,<5901.1032.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1032.0> 2011-07-14 00:19:11 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1207.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:11 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 40 times in the past 38.655892 secs (last seen 1.000225 secs ago PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1218.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1219.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1220.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1221.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1222.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1223.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1218.0> 2011-07-14 00:19:11 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1218.0> 2011-07-14 00:19:11 =============================================================================== ** Generic server <5901.1218.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.9787>}, init, {1310,617151,402534}, "default",#Port<5901.2550>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1218.0> 2011-07-14 00:19:11 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1218.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1031.0>] messages [] links [<5901.49.0>,<5901.1032.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1032.0> 2011-07-14 00:19:11 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1218.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1220.0> 2011-07-14 00:19:11 =============================================================================== ns_1@10.88.165.188:<5901.1220.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:11 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 39 times in the past 38.998557 secs (last seen 0.999947 secs ago PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1230.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1231.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1232.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1233.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1234.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1235.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:11 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:11 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.1230.0> 2011-07-14 00:19:12 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1230.0> 2011-07-14 00:19:12 =============================================================================== ** Generic server <5901.1230.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.9872>}, init, {1310,617151,902533}, "default",#Port<5901.2551>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1230.0> 2011-07-14 00:19:12 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1230.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1031.0>] messages [] links [<5901.49.0>,<5901.1032.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1032.0> 2011-07-14 00:19:12 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1230.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:12 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 41 times in the past 39.655758 secs (last seen 0.999866 secs ago PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1239.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1240.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1241.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1242.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1243.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1244.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1239.0> 2011-07-14 00:19:12 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1239.0> 2011-07-14 00:19:12 =============================================================================== ** Generic server <5901.1239.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.9959>}, init, {1310,617152,404623}, "default",#Port<5901.2552>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1239.0> 2011-07-14 00:19:12 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1239.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1031.0>] messages [] links [<5901.49.0>,<5901.1032.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1032.0> 2011-07-14 00:19:12 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1239.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1241.0> 2011-07-14 00:19:12 =============================================================================== ns_1@10.88.165.188:<5901.1241.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:12 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 40 times in the past 39.998748 secs (last seen 1.000191 secs ago PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1248.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1249.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1250.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1251.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1252.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1032.0> 2011-07-14 00:19:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1253.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1248.0> 2011-07-14 00:19:13 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1248.0> 2011-07-14 00:19:13 =============================================================================== ** Generic server <5901.1248.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.10041>}, init, {1310,617152,901868}, "default",#Port<5901.2553>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1248.0> 2011-07-14 00:19:13 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1248.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1031.0>] messages [] links [<5901.49.0>,<5901.1032.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1032.0> 2011-07-14 00:19:13 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1248.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker SUPERVISOR REPORT <5901.1032.0> 2011-07-14 00:19:13 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext shutdown reason reached_max_restart_intensity pid <5901.1248.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:13 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 42 times in the past 40.658607 secs (last seen 1.002849 secs ago INFO REPORT <5901.1031.0> 2011-07-14 00:19:13 =============================================================================== ns_1@10.88.165.188:<5901.1031.0>:single_bucket_sup:24: per-bucket supervisor for "default" died with reason shutdown SUPERVISOR REPORT <5901.273.0> 2011-07-14 00:19:13 =============================================================================== Reporting supervisor {local,ns_bucket_sup} Child process errorContext child_terminated reason shutdown pid <5901.1031.0> name {per_bucket_sup,"default"} mfargs {single_bucket_sup,start_link,["default"]} restart_type permanent shutdown infinity child_type supervisor PROGRESS REPORT <5901.273.0> 2011-07-14 00:19:13 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<5901.1257.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1259.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:13 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:13 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1260.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1261.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1262.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1263.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1264.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.256.0> 2011-07-14 00:19:13 =============================================================================== ns_1@10.88.165.188:<5901.256.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.88.165.188', [{last_heard,{1310,617153,602286}}, {active_buckets,["default"]}, {ready_buckets,[]}, {replication,[{"default",1.0}]}, {memory, [{total,28369944}, {processes,6697544}, {processes_used,6689560}, {system,21672400}, {atom,814593}, {atom_used,805810}, {binary,486512}, {code,7689836}, {ets,3388992}]}, {system_stats,[{cpu_utilization_rate,2.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,61}, {memory_data,{644874240,573231104,{<5901.80.0>,601112}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 70204 kB\nBuffers: 22008 kB\nCached: 364432 kB\nSwapCached: 0 kB\nActive: 214172 kB\nInactive: 268844 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 70204 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 340 kB\nWriteback: 0 kB\nAnonPages: 96564 kB\nMapped: 11860 kB\nSlab: 36720 kB\nPageTables: 3768 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 545028 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,373178368}, {buffered_memory,22536192}, {free_memory,71888896}, {total_memory,644874240}]}, {statistics, [{wall_clock,{46636,1}}, {context_switches,{26937,0}}, {garbage_collection,{5530,11780856,0}}, {io,{{input,15227344},{output,5801264}}}, {reductions,{3414446,871394}}, {run_queue,0}, {runtime,{480,30}}]}]}, {'ns_1@10.88.23.62', [{last_heard,{1310,617153,612214}}, {active_buckets,["default"]}, {ready_buckets,[]}, {replication,[{"default",1.0}]}, {memory, [{total,29201936}, {processes,7030800}, {processes_used,7004496}, {system,22171136}, {atom,820249}, {atom_used,814132}, {binary,912952}, {code,7741308}, {ets,3402064}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,65}, {memory_data,{644874240,578318336,{<5990.354.0>,601368}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 64872 kB\nBuffers: 21536 kB\nCached: 366340 kB\nSwapCached: 0 kB\nActive: 220284 kB\nInactive: 267932 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 64872 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 524 kB\nWriteback: 0 kB\nAnonPages: 100360 kB\nMapped: 11872 kB\nSlab: 36832 kB\nPageTables: 3760 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 543904 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,375132160}, {buffered_memory,22052864}, {free_memory,66428928}, {total_memory,644874240}]}, {statistics, [{wall_clock,{61502,0}}, {context_switches,{37625,0}}, {garbage_collection,{7762,23736255,0}}, {io,{{input,15945166},{output,6147838}}}, {reductions,{4914559,933715}}, {run_queue,0}, {runtime,{740,30}}]}]}] INFO REPORT <5901.1259.0> 2011-07-14 00:19:13 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1259.0> 2011-07-14 00:19:13 =============================================================================== ** Generic server <5901.1259.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.10127>}, init, {1310,617153,405467}, "default",#Port<5901.2554>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1259.0> 2011-07-14 00:19:13 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1259.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1257.0>] messages [] links [<5901.49.0>,<5901.1258.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6671 SUPERVISOR REPORT <5901.1258.0> 2011-07-14 00:19:13 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1259.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1261.0> 2011-07-14 00:19:13 =============================================================================== ns_1@10.88.165.188:<5901.1261.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:13 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 41 times in the past 40.997224 secs (last seen 0.998476 secs ago PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1279.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1281.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1283.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1284.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1286.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1287.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1279.0> 2011-07-14 00:19:13 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1279.0> 2011-07-14 00:19:13 =============================================================================== ** Generic server <5901.1279.0> terminating ** Last message in was list_vbuckets_prevstate ** When Server state == {state,{interval,#Ref<5901.0.0.10263>}, init, {1310,617153,901117}, "default",#Port<5901.2557>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1279.0> 2011-07-14 00:19:13 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1279.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1257.0>] messages [] links [<5901.49.0>,<5901.1258.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6701 SUPERVISOR REPORT <5901.1258.0> 2011-07-14 00:19:13 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1279.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker CRASH REPORT <5901.1273.0> 2011-07-14 00:19:13 =============================================================================== Crashing process initial_call {ns_janitor,cleanup,['Argument__1','Argument__2']} pid <5901.1273.0> registered_name [] error_info {exit,{{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, {gen_server,call, [{'ns_memcached-default','ns_1@10.88.165.188'}, list_vbuckets_prevstate,30000]}}, [{gen_server,call,3}, {ns_janitor,'-read_existing_map/4-lc$^0/1-0-',2}, {ns_janitor,read_existing_map,4}, {ns_janitor,cleanup,2}, {proc_lib,init_p_do_apply,3}]} ancestors [<5901.304.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<5901.52.0>] messages [] links [<5901.304.0>] dictionary [] trap_exit false status running heap_size 10946 stack_size 24 reductions 224 INFO REPORT <5901.227.0> 2011-07-14 00:19:13 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 42 times in the past 41.008616 secs (last seen 0.011392 secs ago INFO REPORT <5901.304.0> 2011-07-14 00:19:13 =============================================================================== ns_1@10.88.165.188:<5901.304.0>:ns_orchestrator:178: Janitor run exited for bucket "default" with reason {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, [{'ns_memcached-default', 'ns_1@10.88.165.188'}, list_vbuckets_prevstate, 30000]}} PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1289.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1290.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1291.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1292.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1293.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1294.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:14 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:14 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.1289.0> 2011-07-14 00:19:14 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1289.0> 2011-07-14 00:19:14 =============================================================================== ** Generic server <5901.1289.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.10417>}, init, {1310,617153,912570}, "default",#Port<5901.2558>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1289.0> 2011-07-14 00:19:14 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1289.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1257.0>] messages [] links [<5901.49.0>,<5901.1258.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1258.0> 2011-07-14 00:19:14 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1289.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:14 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 43 times in the past 41.667017 secs (last seen 1.00841 secs ago PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1298.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1299.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1300.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1301.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1302.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1303.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1298.0> 2011-07-14 00:19:14 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1298.0> 2011-07-14 00:19:14 =============================================================================== ** Generic server <5901.1298.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.10535>}, init, {1310,617154,413654}, "default",#Port<5901.2560>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1298.0> 2011-07-14 00:19:14 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1298.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1257.0>] messages [] links [<5901.49.0>,<5901.1258.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1258.0> 2011-07-14 00:19:14 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1298.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1300.0> 2011-07-14 00:19:14 =============================================================================== ns_1@10.88.165.188:<5901.1300.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:14 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 43 times in the past 41.998871 secs (last seen 0.990255 secs ago PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1307.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1308.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1309.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1310.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1311.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1312.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1307.0> 2011-07-14 00:19:15 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1307.0> 2011-07-14 00:19:15 =============================================================================== ** Generic server <5901.1307.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.10617>}, init, {1310,617154,902809}, "default",#Port<5901.2561>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1307.0> 2011-07-14 00:19:15 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1307.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1257.0>] messages [] links [<5901.49.0>,<5901.1258.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1258.0> 2011-07-14 00:19:15 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1307.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:15 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 44 times in the past 42.65606 secs (last seen 0.989043 secs ago PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1316.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1317.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1318.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1319.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1320.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1321.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:15 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:15 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.1316.0> 2011-07-14 00:19:15 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1316.0> 2011-07-14 00:19:15 =============================================================================== ** Generic server <5901.1316.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.10699>}, init, {1310,617155,402573}, "default",#Port<5901.2562>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1316.0> 2011-07-14 00:19:15 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1316.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1257.0>] messages [] links [<5901.49.0>,<5901.1258.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1258.0> 2011-07-14 00:19:15 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1316.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1318.0> 2011-07-14 00:19:15 =============================================================================== ns_1@10.88.165.188:<5901.1318.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:15 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 44 times in the past 42.998808 secs (last seen 0.999937 secs ago PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1328.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1329.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1330.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1331.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1332.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1333.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1328.0> 2011-07-14 00:19:16 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1328.0> 2011-07-14 00:19:16 =============================================================================== ** Generic server <5901.1328.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.10784>}, init, {1310,617155,901995}, "default",#Port<5901.2563>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1328.0> 2011-07-14 00:19:16 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1328.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1257.0>] messages [] links [<5901.49.0>,<5901.1258.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1258.0> 2011-07-14 00:19:16 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1328.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:16 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 45 times in the past 43.658758 secs (last seen 1.002698 secs ago PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1337.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1338.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1339.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1340.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1341.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1342.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1337.0> 2011-07-14 00:19:16 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1337.0> 2011-07-14 00:19:16 =============================================================================== ** Generic server <5901.1337.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.10873>}, init, {1310,617156,405289}, "default",#Port<5901.2564>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1337.0> 2011-07-14 00:19:16 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1337.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1257.0>] messages [] links [<5901.49.0>,<5901.1258.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1258.0> 2011-07-14 00:19:16 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1337.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1339.0> 2011-07-14 00:19:16 =============================================================================== ns_1@10.88.165.188:<5901.1339.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:16 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 45 times in the past 43.999455 secs (last seen 1.000647 secs ago PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1346.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:16 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:16 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1347.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1348.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1349.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1350.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1351.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1346.0> 2011-07-14 00:19:17 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1346.0> 2011-07-14 00:19:17 =============================================================================== ** Generic server <5901.1346.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.10949>}, init, {1310,617156,902487}, "default",#Port<5901.2565>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1346.0> 2011-07-14 00:19:17 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1346.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1257.0>] messages [] links [<5901.49.0>,<5901.1258.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1258.0> 2011-07-14 00:19:17 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1346.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:17 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 46 times in the past 44.659021 secs (last seen 1.000263 secs ago PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1357.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1358.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1359.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1360.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1361.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1362.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1357.0> 2011-07-14 00:19:17 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1357.0> 2011-07-14 00:19:17 =============================================================================== ** Generic server <5901.1357.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.11093>}, init, {1310,617157,405608}, "default",#Port<5901.2567>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1357.0> 2011-07-14 00:19:17 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1357.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1257.0>] messages [] links [<5901.49.0>,<5901.1258.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1258.0> 2011-07-14 00:19:17 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1357.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1359.0> 2011-07-14 00:19:17 =============================================================================== ns_1@10.88.165.188:<5901.1359.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:17 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 46 times in the past 44.998901 secs (last seen 0.999446 secs ago PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1369.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1370.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1371.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:17 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:17 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1372.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1373.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1374.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1369.0> 2011-07-14 00:19:18 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1369.0> 2011-07-14 00:19:18 =============================================================================== ** Generic server <5901.1369.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.11174>}, init, {1310,617157,901975}, "default",#Port<5901.2568>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1369.0> 2011-07-14 00:19:18 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1369.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1257.0>] messages [] links [<5901.49.0>,<5901.1258.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1258.0> 2011-07-14 00:19:18 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1369.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:18 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 47 times in the past 45.658905 secs (last seen 0.999884 secs ago PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1378.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1379.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1380.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1381.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1382.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1383.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1378.0> 2011-07-14 00:19:18 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1378.0> 2011-07-14 00:19:18 =============================================================================== ** Generic server <5901.1378.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.11260>}, init, {1310,617158,405528}, "default",#Port<5901.2569>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1378.0> 2011-07-14 00:19:18 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1378.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1257.0>] messages [] links [<5901.49.0>,<5901.1258.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6665 SUPERVISOR REPORT <5901.1258.0> 2011-07-14 00:19:18 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1378.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1380.0> 2011-07-14 00:19:18 =============================================================================== ns_1@10.88.165.188:<5901.1380.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:18 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 47 times in the past 46.000804 secs (last seen 1.001903 secs ago SUPERVISOR REPORT <5901.258.0> 2011-07-14 00:19:18 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call,['ns_memcached-default',topkeys,30000]}} pid <5901.262.0> name hot_keys_keeper mfargs {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <5901.258.0> 2011-07-14 00:19:18 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<5901.1392.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1390.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1393.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1394.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1395.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1396.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1397.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1390.0> 2011-07-14 00:19:19 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1390.0> 2011-07-14 00:19:19 =============================================================================== ** Generic server <5901.1390.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.11359>}, init, {1310,617158,905024}, "default",#Port<5901.2570>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1390.0> 2011-07-14 00:19:19 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1390.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1257.0>] messages [] links [<5901.49.0>,<5901.1258.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1258.0> 2011-07-14 00:19:19 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1390.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:19 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 48 times in the past 46.660822 secs (last seen 1.001917 secs ago PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1401.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:19 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:19 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1402.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1403.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1404.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1405.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1406.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1401.0> 2011-07-14 00:19:19 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1401.0> 2011-07-14 00:19:19 =============================================================================== ** Generic server <5901.1401.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.11444>}, init, {1310,617159,407429}, "default",#Port<5901.2571>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1401.0> 2011-07-14 00:19:19 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1401.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1257.0>] messages [] links [<5901.49.0>,<5901.1258.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1258.0> 2011-07-14 00:19:19 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1401.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1403.0> 2011-07-14 00:19:19 =============================================================================== ns_1@10.88.165.188:<5901.1403.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:19 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 48 times in the past 47.000912 secs (last seen 1.000108 secs ago PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1413.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1414.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1415.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] SUPERVISOR REPORT <5901.258.0> 2011-07-14 00:19:19 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.88.165.188'}, {latest,minute,1}]}} pid <5901.1211.0> name menelaus_web_alerts_srv mfargs {menelaus_web_alerts_srv,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1416.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.258.0> 2011-07-14 00:19:19 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<5901.1418.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1419.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1420.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1413.0> 2011-07-14 00:19:20 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1413.0> 2011-07-14 00:19:20 =============================================================================== ** Generic server <5901.1413.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.11531>}, init, {1310,617159,903991}, "default",#Port<5901.2572>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1413.0> 2011-07-14 00:19:20 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1413.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1257.0>] messages [] links [<5901.49.0>,<5901.1258.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1258.0> 2011-07-14 00:19:20 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1413.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:20 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 49 times in the past 47.66083 secs (last seen 1.000008 secs ago PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1425.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:20 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:20 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1426.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1427.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1428.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1429.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1430.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1425.0> 2011-07-14 00:19:20 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1425.0> 2011-07-14 00:19:20 =============================================================================== ** Generic server <5901.1425.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.11644>}, init, {1310,617160,407396}, "default",#Port<5901.2573>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1425.0> 2011-07-14 00:19:20 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1425.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1257.0>] messages [] links [<5901.49.0>,<5901.1258.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1258.0> 2011-07-14 00:19:20 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1425.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1427.0> 2011-07-14 00:19:20 =============================================================================== ns_1@10.88.165.188:<5901.1427.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:20 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 49 times in the past 48.001205 secs (last seen 1.000293 secs ago PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1434.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1435.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1436.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1437.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1438.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1439.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1434.0> 2011-07-14 00:19:21 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1434.0> 2011-07-14 00:19:21 =============================================================================== ** Generic server <5901.1434.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.11722>}, init, {1310,617160,904263}, "default",#Port<5901.2574>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1434.0> 2011-07-14 00:19:21 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1434.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1257.0>] messages [] links [<5901.49.0>,<5901.1258.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1258.0> 2011-07-14 00:19:21 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1434.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:21 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 50 times in the past 48.660964 secs (last seen 1.000134 secs ago PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1443.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1444.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1445.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:21 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:21 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1446.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1447.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1448.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1443.0> 2011-07-14 00:19:21 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1443.0> 2011-07-14 00:19:21 =============================================================================== ** Generic server <5901.1443.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.11809>}, init, {1310,617161,407555}, "default",#Port<5901.2575>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1443.0> 2011-07-14 00:19:21 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1443.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1257.0>] messages [] links [<5901.49.0>,<5901.1258.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1258.0> 2011-07-14 00:19:21 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1443.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1445.0> 2011-07-14 00:19:21 =============================================================================== ns_1@10.88.165.188:<5901.1445.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:21 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 50 times in the past 48.997329 secs (last seen 0.996124 secs ago PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1455.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1456.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1457.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1458.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1459.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1460.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1455.0> 2011-07-14 00:19:22 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1455.0> 2011-07-14 00:19:22 =============================================================================== ** Generic server <5901.1455.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.11890>}, init, {1310,617161,900423}, "default",#Port<5901.2576>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1455.0> 2011-07-14 00:19:22 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1455.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1257.0>] messages [] links [<5901.49.0>,<5901.1258.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1258.0> 2011-07-14 00:19:22 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1455.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:22 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 51 times in the past 49.657226 secs (last seen 0.996262 secs ago PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1464.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1465.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1466.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1467.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1468.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1469.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1464.0> 2011-07-14 00:19:22 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1464.0> 2011-07-14 00:19:22 =============================================================================== ** Generic server <5901.1464.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.11978>}, init, {1310,617162,403829}, "default",#Port<5901.2577>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1464.0> 2011-07-14 00:19:22 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1464.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1257.0>] messages [] links [<5901.49.0>,<5901.1258.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1258.0> 2011-07-14 00:19:22 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1464.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1466.0> 2011-07-14 00:19:22 =============================================================================== ns_1@10.88.165.188:<5901.1466.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:22 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 51 times in the past 49.997428 secs (last seen 1.000099 secs ago PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1473.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1474.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1475.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:22 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:22 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1476.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1477.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1258.0> 2011-07-14 00:19:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1478.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1473.0> 2011-07-14 00:19:23 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1473.0> 2011-07-14 00:19:23 =============================================================================== ** Generic server <5901.1473.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.12063>}, init, {1310,617162,900536}, "default",#Port<5901.2578>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1473.0> 2011-07-14 00:19:23 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1473.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1257.0>] messages [] links [<5901.49.0>,<5901.1258.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1258.0> 2011-07-14 00:19:23 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1473.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker SUPERVISOR REPORT <5901.1258.0> 2011-07-14 00:19:23 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext shutdown reason reached_max_restart_intensity pid <5901.1473.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:23 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 52 times in the past 50.65715 secs (last seen 0.999924 secs ago INFO REPORT <5901.1257.0> 2011-07-14 00:19:23 =============================================================================== ns_1@10.88.165.188:<5901.1257.0>:single_bucket_sup:24: per-bucket supervisor for "default" died with reason shutdown SUPERVISOR REPORT <5901.273.0> 2011-07-14 00:19:23 =============================================================================== Reporting supervisor {local,ns_bucket_sup} Child process errorContext child_terminated reason shutdown pid <5901.1257.0> name {per_bucket_sup,"default"} mfargs {single_bucket_sup,start_link,["default"]} restart_type permanent shutdown infinity child_type supervisor PROGRESS REPORT <5901.273.0> 2011-07-14 00:19:23 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<5901.1484.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1486.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1487.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1488.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1489.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1490.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1491.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1486.0> 2011-07-14 00:19:23 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1486.0> 2011-07-14 00:19:23 =============================================================================== ** Generic server <5901.1486.0> terminating ** Last message in was list_vbuckets_prevstate ** When Server state == {state,{interval,#Ref<5901.0.0.12201>}, init, {1310,617163,404008}, "default",#Port<5901.2580>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1486.0> 2011-07-14 00:19:23 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1486.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1484.0>] messages [{'$gen_call',{<5901.1488.0>,#Ref<5901.0.0.12309>},{stats,<<>>}}] links [<5901.49.0>,<5901.1485.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6671 SUPERVISOR REPORT <5901.1485.0> 2011-07-14 00:19:23 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1486.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker CRASH REPORT <5901.1498.0> 2011-07-14 00:19:23 =============================================================================== Crashing process initial_call {ns_janitor,cleanup,['Argument__1','Argument__2']} pid <5901.1498.0> registered_name [] error_info {exit,{{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, {gen_server,call, [{'ns_memcached-default','ns_1@10.88.165.188'}, list_vbuckets_prevstate,30000]}}, [{gen_server,call,3}, {ns_janitor,'-read_existing_map/4-lc$^0/1-0-',2}, {ns_janitor,read_existing_map,4}, {ns_janitor,cleanup,2}, {proc_lib,init_p_do_apply,3}]} ancestors [<5901.304.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<5901.52.0>] messages [] links [<5901.304.0>] dictionary [] trap_exit false status running heap_size 10946 stack_size 24 reductions 224 ERROR REPORT <5901.1488.0> 2011-07-14 00:19:23 =============================================================================== ns_1@10.88.165.188:<5901.1488.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:23 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 52 times in the past 50.997249 secs (last seen 0.999821 secs ago INFO REPORT <5901.304.0> 2011-07-14 00:19:23 =============================================================================== ns_1@10.88.165.188:<5901.304.0>:ns_orchestrator:178: Janitor run exited for bucket "default" with reason {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, [{'ns_memcached-default', 'ns_1@10.88.165.188'}, list_vbuckets_prevstate, 30000]}} PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1504.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1505.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1507.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1508.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1510.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1511.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1504.0> 2011-07-14 00:19:24 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1504.0> 2011-07-14 00:19:24 =============================================================================== ** Generic server <5901.1504.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.12319>}, init, {1310,617163,900498}, "default",#Port<5901.2582>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1504.0> 2011-07-14 00:19:24 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1504.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1484.0>] messages [] links [<5901.49.0>,<5901.1485.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.1485.0> 2011-07-14 00:19:24 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1504.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:24 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 53 times in the past 51.65444 secs (last seen 0.99729 secs ago PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1515.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1516.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1517.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1518.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1519.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1520.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:24 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:24 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.1515.0> 2011-07-14 00:19:24 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1515.0> 2011-07-14 00:19:24 =============================================================================== ** Generic server <5901.1515.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.12415>}, init, {1310,617164,400996}, "default",#Port<5901.2583>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1515.0> 2011-07-14 00:19:24 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1515.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1484.0>] messages [] links [<5901.49.0>,<5901.1485.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1485.0> 2011-07-14 00:19:24 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1515.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1517.0> 2011-07-14 00:19:24 =============================================================================== ns_1@10.88.165.188:<5901.1517.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:24 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 53 times in the past 51.997441 secs (last seen 1.000192 secs ago PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1524.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1525.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1526.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1527.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1528.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1529.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1524.0> 2011-07-14 00:19:25 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1524.0> 2011-07-14 00:19:25 =============================================================================== ** Generic server <5901.1524.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.12491>}, init, {1310,617164,900606}, "default",#Port<5901.2584>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1524.0> 2011-07-14 00:19:25 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1524.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1484.0>] messages [] links [<5901.49.0>,<5901.1485.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1485.0> 2011-07-14 00:19:25 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1524.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:25 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 54 times in the past 52.654636 secs (last seen 1.000196 secs ago PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1533.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1534.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1535.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1536.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1537.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1538.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1533.0> 2011-07-14 00:19:25 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1533.0> 2011-07-14 00:19:25 =============================================================================== ** Generic server <5901.1533.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.12579>}, init, {1310,617165,401174}, "default",#Port<5901.2585>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1533.0> 2011-07-14 00:19:25 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1533.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1484.0>] messages [] links [<5901.49.0>,<5901.1485.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1485.0> 2011-07-14 00:19:25 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1533.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1535.0> 2011-07-14 00:19:25 =============================================================================== ns_1@10.88.165.188:<5901.1535.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:25 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 54 times in the past 52.99825 secs (last seen 1.000809 secs ago PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1545.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:25 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:25 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1546.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1547.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1548.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1549.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1550.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1545.0> 2011-07-14 00:19:26 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1545.0> 2011-07-14 00:19:26 =============================================================================== ** Generic server <5901.1545.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.12660>}, init, {1310,617165,901272}, "default",#Port<5901.2586>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1545.0> 2011-07-14 00:19:26 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1545.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1484.0>] messages [] links [<5901.49.0>,<5901.1485.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1485.0> 2011-07-14 00:19:26 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1545.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:26 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 55 times in the past 53.657311 secs (last seen 1.002675 secs ago PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1555.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1556.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1557.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1558.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1559.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1560.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1555.0> 2011-07-14 00:19:26 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1555.0> 2011-07-14 00:19:26 =============================================================================== ** Generic server <5901.1555.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.12793>}, init, {1310,617166,404279}, "default",#Port<5901.2588>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1555.0> 2011-07-14 00:19:26 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1555.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1484.0>] messages [] links [<5901.49.0>,<5901.1485.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1485.0> 2011-07-14 00:19:26 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1555.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1557.0> 2011-07-14 00:19:26 =============================================================================== ns_1@10.88.165.188:<5901.1557.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:26 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 55 times in the past 53.997498 secs (last seen 0.999248 secs ago PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1565.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1566.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1567.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:26 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:26 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1568.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1569.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1570.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1565.0> 2011-07-14 00:19:27 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1565.0> 2011-07-14 00:19:27 =============================================================================== ** Generic server <5901.1565.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.12893>}, init, {1310,617166,900577}, "default",#Port<5901.2589>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1565.0> 2011-07-14 00:19:27 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1565.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1484.0>] messages [] links [<5901.49.0>,<5901.1485.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1485.0> 2011-07-14 00:19:27 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1565.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:27 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 56 times in the past 54.657345 secs (last seen 1.000034 secs ago PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1574.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1575.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1576.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1577.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1578.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1579.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1574.0> 2011-07-14 00:19:27 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1574.0> 2011-07-14 00:19:27 =============================================================================== ** Generic server <5901.1574.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.12975>}, init, {1310,617167,403979}, "default",#Port<5901.2590>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1574.0> 2011-07-14 00:19:27 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1574.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1484.0>] messages [] links [<5901.49.0>,<5901.1485.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1485.0> 2011-07-14 00:19:27 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1574.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1576.0> 2011-07-14 00:19:27 =============================================================================== ns_1@10.88.165.188:<5901.1576.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:27 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 56 times in the past 54.997509 secs (last seen 1.000011 secs ago PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1586.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1587.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1588.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1589.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1590.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1591.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.239.0> 2011-07-14 00:19:28 =============================================================================== ns_1@10.88.165.188:<5901.239.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.1586.0> 2011-07-14 00:19:28 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1586.0> 2011-07-14 00:19:28 =============================================================================== ** Generic server <5901.1586.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.13062>}, init, {1310,617167,901466}, "default",#Port<5901.2591>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1586.0> 2011-07-14 00:19:28 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1586.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1484.0>] messages [] links [<5901.49.0>,<5901.1485.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1485.0> 2011-07-14 00:19:28 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1586.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:28 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 57 times in the past 55.654637 secs (last seen 0.997292 secs ago PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1597.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1598.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1599.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1600.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1601.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1602.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:28 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:28 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.1597.0> 2011-07-14 00:19:28 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1597.0> 2011-07-14 00:19:28 =============================================================================== ** Generic server <5901.1597.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.13158>}, init, {1310,617168,401177}, "default",#Port<5901.2593>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1597.0> 2011-07-14 00:19:28 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1597.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1484.0>] messages [] links [<5901.49.0>,<5901.1485.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6665 SUPERVISOR REPORT <5901.1485.0> 2011-07-14 00:19:28 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1597.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1599.0> 2011-07-14 00:19:28 =============================================================================== ns_1@10.88.165.188:<5901.1599.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:28 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 57 times in the past 55.997705 secs (last seen 1.000196 secs ago PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1610.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1611.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] SUPERVISOR REPORT <5901.258.0> 2011-07-14 00:19:28 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.88.165.188'}, {latest,minute,1}]}} pid <5901.1418.0> name menelaus_web_alerts_srv mfargs {menelaus_web_alerts_srv,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1614.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.258.0> 2011-07-14 00:19:28 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<5901.1615.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1616.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1618.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1619.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1610.0> 2011-07-14 00:19:29 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1610.0> 2011-07-14 00:19:29 =============================================================================== ** Generic server <5901.1610.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.13327>}, init, {1310,617168,906160}, "default",#Port<5901.2595>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1610.0> 2011-07-14 00:19:29 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1610.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1484.0>] messages [] links [<5901.49.0>,<5901.1485.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.1485.0> 2011-07-14 00:19:29 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1610.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:29 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 58 times in the past 56.657514 secs (last seen 1.002877 secs ago PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1623.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1624.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1625.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1626.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1627.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1628.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1623.0> 2011-07-14 00:19:29 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1623.0> 2011-07-14 00:19:29 =============================================================================== ** Generic server <5901.1623.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.13427>}, init, {1310,617169,404089}, "default",#Port<5901.2596>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1623.0> 2011-07-14 00:19:29 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1623.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1484.0>] messages [] links [<5901.49.0>,<5901.1485.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1485.0> 2011-07-14 00:19:29 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1623.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1625.0> 2011-07-14 00:19:29 =============================================================================== ns_1@10.88.165.188:<5901.1625.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:29 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 58 times in the past 56.997511 secs (last seen 0.999806 secs ago PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1636.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1637.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1638.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1639.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1640.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1641.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:29 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:29 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.1636.0> 2011-07-14 00:19:30 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1636.0> 2011-07-14 00:19:30 =============================================================================== ** Generic server <5901.1636.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.13528>}, init, {1310,617169,900633}, "default",#Port<5901.2597>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1636.0> 2011-07-14 00:19:30 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1636.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1484.0>] messages [] links [<5901.49.0>,<5901.1485.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1485.0> 2011-07-14 00:19:30 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1636.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:30 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 59 times in the past 57.654681 secs (last seen 0.997167 secs ago PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1645.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1646.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1647.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1648.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1649.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1650.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1645.0> 2011-07-14 00:19:30 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1645.0> 2011-07-14 00:19:30 =============================================================================== ** Generic server <5901.1645.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.13614>}, init, {1310,617170,404198}, "default",#Port<5901.2598>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1645.0> 2011-07-14 00:19:30 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1645.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1484.0>] messages [] links [<5901.49.0>,<5901.1485.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1485.0> 2011-07-14 00:19:30 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1645.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1647.0> 2011-07-14 00:19:30 =============================================================================== ns_1@10.88.165.188:<5901.1647.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:30 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 59 times in the past 57.99774 secs (last seen 1.000229 secs ago PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1654.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1655.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1656.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1657.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1658.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1659.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1654.0> 2011-07-14 00:19:31 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1654.0> 2011-07-14 00:19:31 =============================================================================== ** Generic server <5901.1654.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.13697>}, init, {1310,617170,900855}, "default",#Port<5901.2599>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1654.0> 2011-07-14 00:19:31 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1654.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1484.0>] messages [] links [<5901.49.0>,<5901.1485.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1485.0> 2011-07-14 00:19:31 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1654.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:31 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 60 times in the past 58.65761 secs (last seen 1.002929 secs ago PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1663.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:31 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:31 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1664.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1665.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1666.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1667.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1668.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1663.0> 2011-07-14 00:19:31 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1663.0> 2011-07-14 00:19:31 =============================================================================== ** Generic server <5901.1663.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.13779>}, init, {1310,617171,405260}, "default",#Port<5901.2600>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1663.0> 2011-07-14 00:19:31 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1663.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1484.0>] messages [] links [<5901.49.0>,<5901.1485.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1485.0> 2011-07-14 00:19:31 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1663.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1665.0> 2011-07-14 00:19:31 =============================================================================== ns_1@10.88.165.188:<5901.1665.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:31 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 60 times in the past 58.997689 secs (last seen 0.999949 secs ago PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1675.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1676.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1677.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1678.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1679.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1680.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1675.0> 2011-07-14 00:19:32 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1675.0> 2011-07-14 00:19:32 =============================================================================== ** Generic server <5901.1675.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.13866>}, init, {1310,617171,900818}, "default",#Port<5901.2601>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1675.0> 2011-07-14 00:19:32 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1675.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1484.0>] messages [] links [<5901.49.0>,<5901.1485.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1485.0> 2011-07-14 00:19:32 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1675.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:32 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 61 times in the past 59.657586 secs (last seen 0.999976 secs ago PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1685.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1686.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1687.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:32 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:32 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1688.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1689.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1690.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1685.0> 2011-07-14 00:19:32 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1685.0> 2011-07-14 00:19:32 =============================================================================== ** Generic server <5901.1685.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.13992>}, init, {1310,617172,404155}, "default",#Port<5901.2603>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1685.0> 2011-07-14 00:19:32 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1685.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1484.0>] messages [] links [<5901.49.0>,<5901.1485.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1485.0> 2011-07-14 00:19:32 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1685.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1687.0> 2011-07-14 00:19:32 =============================================================================== ns_1@10.88.165.188:<5901.1687.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:32 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 61 times in the past 59.997815 secs (last seen 1.000126 secs ago PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1695.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1696.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1697.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1698.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1699.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1700.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1695.0> 2011-07-14 00:19:33 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1695.0> 2011-07-14 00:19:33 =============================================================================== ** Generic server <5901.1695.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.14088>}, init, {1310,617172,900891}, "default",#Port<5901.2604>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1695.0> 2011-07-14 00:19:33 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1695.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1484.0>] messages [] links [<5901.49.0>,<5901.1485.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1485.0> 2011-07-14 00:19:33 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1695.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:33 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 62 times in the past 60.655002 secs (last seen 0.997416 secs ago PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1704.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1705.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1706.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1707.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1708.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1485.0> 2011-07-14 00:19:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1709.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1704.0> 2011-07-14 00:19:33 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1704.0> 2011-07-14 00:19:33 =============================================================================== ** Generic server <5901.1704.0> terminating ** Last message in was list_vbuckets_prevstate ** When Server state == {state,{interval,#Ref<5901.0.0.14177>}, init, {1310,617173,401547}, "default",#Port<5901.2605>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1704.0> 2011-07-14 00:19:33 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1704.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1484.0>] messages [{'$gen_call',{<5901.1706.0>,#Ref<5901.0.0.14278>},{stats,<<>>}}] links [<5901.49.0>,<5901.1485.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6671 SUPERVISOR REPORT <5901.1485.0> 2011-07-14 00:19:33 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1704.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker SUPERVISOR REPORT <5901.1485.0> 2011-07-14 00:19:33 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext shutdown reason reached_max_restart_intensity pid <5901.1704.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker CRASH REPORT <5901.1716.0> 2011-07-14 00:19:33 =============================================================================== Crashing process initial_call {ns_janitor,cleanup,['Argument__1','Argument__2']} pid <5901.1716.0> registered_name [] error_info {exit,{{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, {gen_server,call, [{'ns_memcached-default','ns_1@10.88.165.188'}, list_vbuckets_prevstate,30000]}}, [{gen_server,call,3}, {ns_janitor,'-read_existing_map/4-lc$^0/1-0-',2}, {ns_janitor,read_existing_map,4}, {ns_janitor,cleanup,2}, {proc_lib,init_p_do_apply,3}]} ancestors [<5901.304.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<5901.52.0>] messages [] links [<5901.304.0>] dictionary [] trap_exit false status running heap_size 10946 stack_size 24 reductions 224 ERROR REPORT <5901.1706.0> 2011-07-14 00:19:33 =============================================================================== ns_1@10.88.165.188:<5901.1706.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:33 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 62 times in the past 60.997239 secs (last seen 0.999424 secs ago INFO REPORT <5901.304.0> 2011-07-14 00:19:33 =============================================================================== ns_1@10.88.165.188:<5901.304.0>:ns_orchestrator:178: Janitor run exited for bucket "default" with reason {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, [{'ns_memcached-default', 'ns_1@10.88.165.188'}, list_vbuckets_prevstate, 30000]}} INFO REPORT <5901.1484.0> 2011-07-14 00:19:33 =============================================================================== ns_1@10.88.165.188:<5901.1484.0>:single_bucket_sup:24: per-bucket supervisor for "default" died with reason shutdown SUPERVISOR REPORT <5901.273.0> 2011-07-14 00:19:33 =============================================================================== Reporting supervisor {local,ns_bucket_sup} Child process errorContext child_terminated reason shutdown pid <5901.1484.0> name {per_bucket_sup,"default"} mfargs {single_bucket_sup,start_link,["default"]} restart_type permanent shutdown infinity child_type supervisor PROGRESS REPORT <5901.273.0> 2011-07-14 00:19:33 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<5901.1722.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1724.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:33 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:33 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1725.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1727.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1728.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1730.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1731.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1724.0> 2011-07-14 00:19:34 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1724.0> 2011-07-14 00:19:34 =============================================================================== ** Generic server <5901.1724.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.14288>}, init, {1310,617173,900634}, "default",#Port<5901.2607>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1724.0> 2011-07-14 00:19:34 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1724.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1722.0>] messages [] links [<5901.49.0>,<5901.1723.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.1723.0> 2011-07-14 00:19:34 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1724.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:34 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 63 times in the past 61.657707 secs (last seen 1.002705 secs ago PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1735.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1736.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1737.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1738.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1739.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1740.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1735.0> 2011-07-14 00:19:34 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1735.0> 2011-07-14 00:19:34 =============================================================================== ** Generic server <5901.1735.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.14384>}, init, {1310,617174,404272}, "default",#Port<5901.2608>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1735.0> 2011-07-14 00:19:34 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1735.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1722.0>] messages [] links [<5901.49.0>,<5901.1723.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1723.0> 2011-07-14 00:19:34 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1735.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1737.0> 2011-07-14 00:19:34 =============================================================================== ns_1@10.88.165.188:<5901.1737.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:34 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 63 times in the past 61.998226 secs (last seen 1.000987 secs ago PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1744.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1745.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1746.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1747.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1748.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1749.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1744.0> 2011-07-14 00:19:35 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1744.0> 2011-07-14 00:19:35 =============================================================================== ** Generic server <5901.1744.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.14466>}, init, {1310,617174,901878}, "default",#Port<5901.2609>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1744.0> 2011-07-14 00:19:35 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1744.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1722.0>] messages [] links [<5901.49.0>,<5901.1723.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1723.0> 2011-07-14 00:19:35 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1744.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:35 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 64 times in the past 62.657877 secs (last seen 1.00017 secs ago PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1754.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1755.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1756.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:35 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:35 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1757.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1758.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1759.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1754.0> 2011-07-14 00:19:35 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1754.0> 2011-07-14 00:19:35 =============================================================================== ** Generic server <5901.1754.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.14585>}, init, {1310,617175,404442}, "default",#Port<5901.2611>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1754.0> 2011-07-14 00:19:35 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1754.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1722.0>] messages [] links [<5901.49.0>,<5901.1723.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1723.0> 2011-07-14 00:19:35 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1754.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1756.0> 2011-07-14 00:19:35 =============================================================================== ns_1@10.88.165.188:<5901.1756.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:35 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 64 times in the past 62.997884 secs (last seen 0.999658 secs ago PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1766.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1767.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1768.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1769.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1770.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1771.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1766.0> 2011-07-14 00:19:36 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1766.0> 2011-07-14 00:19:36 =============================================================================== ** Generic server <5901.1766.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.14670>}, init, {1310,617175,901020}, "default",#Port<5901.2612>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1766.0> 2011-07-14 00:19:36 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1766.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1722.0>] messages [] links [<5901.49.0>,<5901.1723.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1723.0> 2011-07-14 00:19:36 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1766.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:36 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 65 times in the past 63.65771 secs (last seen 0.999833 secs ago PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1776.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1777.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1778.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1779.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1780.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1781.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1776.0> 2011-07-14 00:19:36 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1776.0> 2011-07-14 00:19:36 =============================================================================== ** Generic server <5901.1776.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.14778>}, init, {1310,617176,404186}, "default",#Port<5901.2613>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1776.0> 2011-07-14 00:19:36 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1776.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1722.0>] messages [] links [<5901.49.0>,<5901.1723.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1723.0> 2011-07-14 00:19:36 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1776.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1778.0> 2011-07-14 00:19:36 =============================================================================== ns_1@10.88.165.188:<5901.1778.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:36 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 65 times in the past 63.998174 secs (last seen 1.00029 secs ago PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1785.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:36 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:36 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1786.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1787.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1788.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1789.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1790.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1785.0> 2011-07-14 00:19:37 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1785.0> 2011-07-14 00:19:37 =============================================================================== ** Generic server <5901.1785.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.14854>}, init, {1310,617176,902327}, "default",#Port<5901.2614>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1785.0> 2011-07-14 00:19:37 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1785.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1722.0>] messages [] links [<5901.49.0>,<5901.1723.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1723.0> 2011-07-14 00:19:37 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1785.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:37 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 66 times in the past 64.65786 secs (last seen 1.00015 secs ago PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1794.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1795.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1796.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1797.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1798.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1799.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1794.0> 2011-07-14 00:19:37 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1794.0> 2011-07-14 00:19:37 =============================================================================== ** Generic server <5901.1794.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.14943>}, init, {1310,617177,404407}, "default",#Port<5901.2615>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1794.0> 2011-07-14 00:19:37 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1794.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1722.0>] messages [] links [<5901.49.0>,<5901.1723.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1723.0> 2011-07-14 00:19:37 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1794.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1796.0> 2011-07-14 00:19:37 =============================================================================== ns_1@10.88.165.188:<5901.1796.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:37 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 66 times in the past 64.998036 secs (last seen 0.999862 secs ago PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1806.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1807.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1808.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1809.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1810.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1811.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:37 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:37 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.1806.0> 2011-07-14 00:19:38 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1806.0> 2011-07-14 00:19:38 =============================================================================== ** Generic server <5901.1806.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.15024>}, init, {1310,617177,901188}, "default",#Port<5901.2616>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1806.0> 2011-07-14 00:19:38 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1806.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1722.0>] messages [] links [<5901.49.0>,<5901.1723.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1723.0> 2011-07-14 00:19:38 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1806.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:38 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 67 times in the past 65.657873 secs (last seen 1.000013 secs ago PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1816.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1817.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1818.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1819.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1820.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1821.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1816.0> 2011-07-14 00:19:38 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1816.0> 2011-07-14 00:19:38 =============================================================================== ** Generic server <5901.1816.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.15154>}, init, {1310,617178,404443}, "default",#Port<5901.2618>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1816.0> 2011-07-14 00:19:38 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1816.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1722.0>] messages [] links [<5901.49.0>,<5901.1723.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6665 SUPERVISOR REPORT <5901.1723.0> 2011-07-14 00:19:38 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1816.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1818.0> 2011-07-14 00:19:38 =============================================================================== ns_1@10.88.165.188:<5901.1818.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:38 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 67 times in the past 65.99811 secs (last seen 1.000074 secs ago PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1828.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1830.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1831.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1832.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1833.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1834.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1828.0> 2011-07-14 00:19:39 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1828.0> 2011-07-14 00:19:39 =============================================================================== ** Generic server <5901.1828.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.15252>}, init, {1310,617178,901240}, "default",#Port<5901.2619>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1828.0> 2011-07-14 00:19:39 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1828.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1722.0>] messages [] links [<5901.49.0>,<5901.1723.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.1723.0> 2011-07-14 00:19:39 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1828.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:39 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 68 times in the past 66.657902 secs (last seen 1.000029 secs ago PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1840.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1841.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1842.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:39 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:39 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1843.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1844.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1845.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1840.0> 2011-07-14 00:19:39 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1840.0> 2011-07-14 00:19:39 =============================================================================== ** Generic server <5901.1840.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.15363>}, init, {1310,617179,404380}, "default",#Port<5901.2620>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1840.0> 2011-07-14 00:19:39 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1840.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1722.0>] messages [] links [<5901.49.0>,<5901.1723.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1723.0> 2011-07-14 00:19:39 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1840.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1842.0> 2011-07-14 00:19:39 =============================================================================== ns_1@10.88.165.188:<5901.1842.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:39 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 68 times in the past 66.998194 secs (last seen 1.000084 secs ago PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1852.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1853.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1854.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1855.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1856.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1857.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1852.0> 2011-07-14 00:19:40 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1852.0> 2011-07-14 00:19:40 =============================================================================== ** Generic server <5901.1852.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.15449>}, init, {1310,617179,901535}, "default",#Port<5901.2621>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1852.0> 2011-07-14 00:19:40 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1852.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1722.0>] messages [] links [<5901.49.0>,<5901.1723.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1723.0> 2011-07-14 00:19:40 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1852.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:40 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 69 times in the past 67.655452 secs (last seen 0.99755 secs ago PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1861.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1862.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1863.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1864.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1865.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1866.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1861.0> 2011-07-14 00:19:40 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1861.0> 2011-07-14 00:19:40 =============================================================================== ** Generic server <5901.1861.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.15537>}, init, {1310,617180,402037}, "default",#Port<5901.2622>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1861.0> 2011-07-14 00:19:40 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1861.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1722.0>] messages [] links [<5901.49.0>,<5901.1723.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1723.0> 2011-07-14 00:19:40 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1861.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1863.0> 2011-07-14 00:19:40 =============================================================================== ns_1@10.88.165.188:<5901.1863.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:40 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 69 times in the past 67.99836 secs (last seen 1.000166 secs ago PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1870.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1871.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1872.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:40 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:40 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1873.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1874.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1875.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1870.0> 2011-07-14 00:19:41 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1870.0> 2011-07-14 00:19:41 =============================================================================== ** Generic server <5901.1870.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.15613>}, init, {1310,617180,901315}, "default",#Port<5901.2623>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1870.0> 2011-07-14 00:19:41 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1870.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1722.0>] messages [] links [<5901.49.0>,<5901.1723.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1723.0> 2011-07-14 00:19:41 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1870.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:41 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 70 times in the past 68.655556 secs (last seen 1.000104 secs ago PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1880.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1881.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1882.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1883.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1884.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1885.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1880.0> 2011-07-14 00:19:41 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1880.0> 2011-07-14 00:19:41 =============================================================================== ** Generic server <5901.1880.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.15735>}, init, {1310,617181,402092}, "default",#Port<5901.2625>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1880.0> 2011-07-14 00:19:41 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1880.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1722.0>] messages [] links [<5901.49.0>,<5901.1723.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1723.0> 2011-07-14 00:19:41 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1880.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1882.0> 2011-07-14 00:19:41 =============================================================================== ns_1@10.88.165.188:<5901.1882.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:41 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 70 times in the past 68.99824 secs (last seen 0.99988 secs ago PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1892.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1893.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1894.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1895.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1896.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1897.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1892.0> 2011-07-14 00:19:42 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1892.0> 2011-07-14 00:19:42 =============================================================================== ** Generic server <5901.1892.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.15818>}, init, {1310,617181,901418}, "default",#Port<5901.2626>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1892.0> 2011-07-14 00:19:42 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1892.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1722.0>] messages [] links [<5901.49.0>,<5901.1723.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1723.0> 2011-07-14 00:19:42 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1892.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:42 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 71 times in the past 69.658124 secs (last seen 1.002568 secs ago PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1902.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:42 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:42 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1903.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1904.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1905.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1906.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1907.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1902.0> 2011-07-14 00:19:42 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1902.0> 2011-07-14 00:19:42 =============================================================================== ** Generic server <5901.1902.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.15925>}, init, {1310,617182,404572}, "default",#Port<5901.2627>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1902.0> 2011-07-14 00:19:42 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1902.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1722.0>] messages [] links [<5901.49.0>,<5901.1723.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1723.0> 2011-07-14 00:19:42 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1902.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1904.0> 2011-07-14 00:19:42 =============================================================================== ns_1@10.88.165.188:<5901.1904.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:42 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 71 times in the past 69.998352 secs (last seen 1.000112 secs ago PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1911.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1912.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1913.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1914.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1915.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1916.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1911.0> 2011-07-14 00:19:43 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1911.0> 2011-07-14 00:19:43 =============================================================================== ** Generic server <5901.1911.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.16007>}, init, {1310,617182,901622}, "default",#Port<5901.2628>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1911.0> 2011-07-14 00:19:43 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1911.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1722.0>] messages [] links [<5901.49.0>,<5901.1723.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1723.0> 2011-07-14 00:19:43 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1911.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:43 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 72 times in the past 70.658226 secs (last seen 1.000102 secs ago PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1920.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1921.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1922.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:43 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:43 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1923.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1924.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1925.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1920.0> 2011-07-14 00:19:43 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1920.0> 2011-07-14 00:19:43 =============================================================================== ** Generic server <5901.1920.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.16089>}, init, {1310,617183,404799}, "default",#Port<5901.2629>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1920.0> 2011-07-14 00:19:43 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1920.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1722.0>] messages [] links [<5901.49.0>,<5901.1723.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6671 SUPERVISOR REPORT <5901.1723.0> 2011-07-14 00:19:43 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1920.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1922.0> 2011-07-14 00:19:43 =============================================================================== ns_1@10.88.165.188:<5901.1922.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:43 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 72 times in the past 70.997243 secs (last seen 0.998891 secs ago PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1938.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1940.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1942.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1943.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1944.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1723.0> 2011-07-14 00:19:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1945.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1938.0> 2011-07-14 00:19:43 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1938.0> 2011-07-14 00:19:43 =============================================================================== ** Generic server <5901.1938.0> terminating ** Last message in was list_vbuckets_prevstate ** When Server state == {state,{interval,#Ref<5901.0.0.16204>}, init, {1310,617183,900844}, "default",#Port<5901.2632>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1938.0> 2011-07-14 00:19:43 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1938.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1722.0>] messages [] links [<5901.49.0>,<5901.1723.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6701 SUPERVISOR REPORT <5901.1723.0> 2011-07-14 00:19:43 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1938.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker SUPERVISOR REPORT <5901.1723.0> 2011-07-14 00:19:43 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext shutdown reason reached_max_restart_intensity pid <5901.1938.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker CRASH REPORT <5901.1932.0> 2011-07-14 00:19:43 =============================================================================== Crashing process initial_call {ns_janitor,cleanup,['Argument__1','Argument__2']} pid <5901.1932.0> registered_name [] error_info {exit,{{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, {gen_server,call, [{'ns_memcached-default','ns_1@10.88.165.188'}, list_vbuckets_prevstate,30000]}}, [{gen_server,call,3}, {ns_janitor,'-read_existing_map/4-lc$^0/1-0-',2}, {ns_janitor,read_existing_map,4}, {ns_janitor,cleanup,2}, {proc_lib,init_p_do_apply,3}]} ancestors [<5901.304.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<5901.52.0>] messages [] links [<5901.304.0>] dictionary [] trap_exit false status running heap_size 10946 stack_size 24 reductions 224 INFO REPORT <5901.227.0> 2011-07-14 00:19:43 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 73 times in the past 71.007769 secs (last seen 0.010526 secs ago INFO REPORT <5901.304.0> 2011-07-14 00:19:43 =============================================================================== ns_1@10.88.165.188:<5901.304.0>:ns_orchestrator:178: Janitor run exited for bucket "default" with reason {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, [{'ns_memcached-default', 'ns_1@10.88.165.188'}, list_vbuckets_prevstate, 30000]}} INFO REPORT <5901.1722.0> 2011-07-14 00:19:43 =============================================================================== ns_1@10.88.165.188:<5901.1722.0>:single_bucket_sup:24: per-bucket supervisor for "default" died with reason shutdown SUPERVISOR REPORT <5901.273.0> 2011-07-14 00:19:43 =============================================================================== Reporting supervisor {local,ns_bucket_sup} Child process errorContext child_terminated reason shutdown pid <5901.1722.0> name {per_bucket_sup,"default"} mfargs {single_bucket_sup,start_link,["default"]} restart_type permanent shutdown infinity child_type supervisor PROGRESS REPORT <5901.273.0> 2011-07-14 00:19:43 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<5901.1947.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1949.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1950.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] SUPERVISOR REPORT <5901.258.0> 2011-07-14 00:19:43 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.88.165.188'}, {latest,minute,1}]}} pid <5901.1615.0> name menelaus_web_alerts_srv mfargs {menelaus_web_alerts_srv,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1952.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.258.0> 2011-07-14 00:19:43 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<5901.1953.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1954.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1955.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1956.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1949.0> 2011-07-14 00:19:44 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1949.0> 2011-07-14 00:19:44 =============================================================================== ** Generic server <5901.1949.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.16357>}, init, {1310,617183,910228}, "default",#Port<5901.2633>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1949.0> 2011-07-14 00:19:44 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1949.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1947.0>] messages [] links [<5901.49.0>,<5901.1948.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1948.0> 2011-07-14 00:19:44 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1949.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:44 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 73 times in the past 71.666276 secs (last seen 1.00805 secs ago PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1960.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1961.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1962.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1963.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1964.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1965.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1960.0> 2011-07-14 00:19:44 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1960.0> 2011-07-14 00:19:44 =============================================================================== ** Generic server <5901.1960.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.16445>}, init, {1310,617184,412760}, "default",#Port<5901.2634>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1960.0> 2011-07-14 00:19:44 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1960.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1947.0>] messages [] links [<5901.49.0>,<5901.1948.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1948.0> 2011-07-14 00:19:44 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1960.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1962.0> 2011-07-14 00:19:44 =============================================================================== ns_1@10.88.165.188:<5901.1962.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:44 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 74 times in the past 71.998465 secs (last seen 0.990696 secs ago PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1969.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:44 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:44 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1970.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1971.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1972.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1973.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1974.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1969.0> 2011-07-14 00:19:45 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1969.0> 2011-07-14 00:19:45 =============================================================================== ** Generic server <5901.1969.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.16521>}, init, {1310,617184,901454}, "default",#Port<5901.2635>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1969.0> 2011-07-14 00:19:45 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1969.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1947.0>] messages [] links [<5901.49.0>,<5901.1948.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1948.0> 2011-07-14 00:19:45 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1969.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:45 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 74 times in the past 72.655469 secs (last seen 0.989193 secs ago PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1979.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1980.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1981.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1982.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1983.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1984.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1979.0> 2011-07-14 00:19:45 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1979.0> 2011-07-14 00:19:45 =============================================================================== ** Generic server <5901.1979.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.16629>}, init, {1310,617185,401956}, "default",#Port<5901.2636>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1979.0> 2011-07-14 00:19:45 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1979.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1947.0>] messages [] links [<5901.49.0>,<5901.1948.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1948.0> 2011-07-14 00:19:45 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1979.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.1981.0> 2011-07-14 00:19:45 =============================================================================== ns_1@10.88.165.188:<5901.1981.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:45 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 75 times in the past 72.998498 secs (last seen 1.000033 secs ago PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1991.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1992.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1993.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:45 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:45 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1994.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1995.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.1996.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.1991.0> 2011-07-14 00:19:46 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.1991.0> 2011-07-14 00:19:46 =============================================================================== ** Generic server <5901.1991.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.16710>}, init, {1310,617185,901554}, "default",#Port<5901.2637>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.1991.0> 2011-07-14 00:19:46 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.1991.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1947.0>] messages [] links [<5901.49.0>,<5901.1948.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1948.0> 2011-07-14 00:19:46 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.1991.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:46 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 75 times in the past 73.655719 secs (last seen 1.00025 secs ago PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2000.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2001.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2002.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2003.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2004.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2005.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2000.0> 2011-07-14 00:19:46 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2000.0> 2011-07-14 00:19:46 =============================================================================== ** Generic server <5901.2000.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.16796>}, init, {1310,617186,402336}, "default",#Port<5901.2638>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2000.0> 2011-07-14 00:19:46 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2000.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1947.0>] messages [] links [<5901.49.0>,<5901.1948.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1948.0> 2011-07-14 00:19:46 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2000.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2002.0> 2011-07-14 00:19:46 =============================================================================== ns_1@10.88.165.188:<5901.2002.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:46 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 76 times in the past 73.998722 secs (last seen 1.000224 secs ago PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2009.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2010.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2011.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2012.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2013.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2014.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2009.0> 2011-07-14 00:19:47 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2009.0> 2011-07-14 00:19:47 =============================================================================== ** Generic server <5901.2009.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.16880>}, init, {1310,617186,901847}, "default",#Port<5901.2641>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2009.0> 2011-07-14 00:19:47 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2009.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1947.0>] messages [] links [<5901.49.0>,<5901.1948.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1948.0> 2011-07-14 00:19:47 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2009.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:47 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 76 times in the past 74.658626 secs (last seen 1.002907 secs ago PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2019.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:47 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:47 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2020.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2021.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2022.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2023.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2024.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2019.0> 2011-07-14 00:19:47 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2019.0> 2011-07-14 00:19:47 =============================================================================== ** Generic server <5901.2019.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.16998>}, init, {1310,617187,405199}, "default",#Port<5901.2643>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2019.0> 2011-07-14 00:19:47 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2019.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1947.0>] messages [] links [<5901.49.0>,<5901.1948.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1948.0> 2011-07-14 00:19:47 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2019.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2021.0> 2011-07-14 00:19:47 =============================================================================== ns_1@10.88.165.188:<5901.2021.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:47 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 77 times in the past 74.998695 secs (last seen 0.999973 secs ago PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2031.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2032.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2033.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2034.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2035.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2036.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2031.0> 2011-07-14 00:19:48 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2031.0> 2011-07-14 00:19:48 =============================================================================== ** Generic server <5901.2031.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.17090>}, init, {1310,617187,901764}, "default",#Port<5901.2644>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2031.0> 2011-07-14 00:19:48 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2031.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1947.0>] messages [] links [<5901.49.0>,<5901.1948.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1948.0> 2011-07-14 00:19:48 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2031.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:48 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 77 times in the past 75.655778 secs (last seen 0.997152 secs ago PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2040.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2041.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2042.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2043.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2044.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2045.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:48 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:48 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.2040.0> 2011-07-14 00:19:48 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2040.0> 2011-07-14 00:19:48 =============================================================================== ** Generic server <5901.2040.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.17176>}, init, {1310,617188,402227}, "default",#Port<5901.2645>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2040.0> 2011-07-14 00:19:48 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2040.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1947.0>] messages [] links [<5901.49.0>,<5901.1948.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6665 SUPERVISOR REPORT <5901.1948.0> 2011-07-14 00:19:48 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2040.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2042.0> 2011-07-14 00:19:48 =============================================================================== ns_1@10.88.165.188:<5901.2042.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:48 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 78 times in the past 75.99867 secs (last seen 0.999975 secs ago PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2053.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2054.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2056.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2057.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2058.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2059.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2053.0> 2011-07-14 00:19:49 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2053.0> 2011-07-14 00:19:49 =============================================================================== ** Generic server <5901.2053.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.17288>}, init, {1310,617188,901720}, "default",#Port<5901.2646>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2053.0> 2011-07-14 00:19:49 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2053.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1947.0>] messages [] links [<5901.49.0>,<5901.1948.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.1948.0> 2011-07-14 00:19:49 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2053.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:49 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 78 times in the past 76.655773 secs (last seen 0.999995 secs ago PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2064.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2065.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2066.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2067.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2068.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2069.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2064.0> 2011-07-14 00:19:49 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2064.0> 2011-07-14 00:19:49 =============================================================================== ** Generic server <5901.2064.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.17386>}, init, {1310,617189,402264}, "default",#Port<5901.2647>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2064.0> 2011-07-14 00:19:49 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2064.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1947.0>] messages [] links [<5901.49.0>,<5901.1948.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1948.0> 2011-07-14 00:19:49 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2064.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2066.0> 2011-07-14 00:19:49 =============================================================================== ns_1@10.88.165.188:<5901.2066.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:49 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 79 times in the past 76.998752 secs (last seen 1.000082 secs ago PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2076.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2077.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2078.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:49 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:49 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2079.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2080.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2081.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2076.0> 2011-07-14 00:19:50 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2076.0> 2011-07-14 00:19:50 =============================================================================== ** Generic server <5901.2076.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.17467>}, init, {1310,617189,901711}, "default",#Port<5901.2648>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2076.0> 2011-07-14 00:19:50 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2076.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1947.0>] messages [] links [<5901.49.0>,<5901.1948.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1948.0> 2011-07-14 00:19:50 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2076.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:50 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 79 times in the past 77.655898 secs (last seen 1.000125 secs ago PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2086.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2087.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2088.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2089.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2090.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2091.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2086.0> 2011-07-14 00:19:50 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2086.0> 2011-07-14 00:19:50 =============================================================================== ** Generic server <5901.2086.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.17598>}, init, {1310,617190,402450}, "default",#Port<5901.2651>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2086.0> 2011-07-14 00:19:50 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2086.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1947.0>] messages [] links [<5901.49.0>,<5901.1948.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1948.0> 2011-07-14 00:19:50 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2086.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2088.0> 2011-07-14 00:19:50 =============================================================================== ns_1@10.88.165.188:<5901.2088.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:50 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 80 times in the past 77.999016 secs (last seen 1.000264 secs ago PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2095.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2096.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2097.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2098.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2099.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2100.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2095.0> 2011-07-14 00:19:51 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2095.0> 2011-07-14 00:19:51 =============================================================================== ** Generic server <5901.2095.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.17680>}, init, {1310,617190,902939}, "default",#Port<5901.2652>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2095.0> 2011-07-14 00:19:51 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2095.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1947.0>] messages [] links [<5901.49.0>,<5901.1948.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1948.0> 2011-07-14 00:19:51 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2095.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:51 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 80 times in the past 78.658918 secs (last seen 1.00302 secs ago PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2104.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:51 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:51 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2105.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2106.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2107.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2108.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2109.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2104.0> 2011-07-14 00:19:51 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2104.0> 2011-07-14 00:19:51 =============================================================================== ** Generic server <5901.2104.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.17763>}, init, {1310,617191,405455}, "default",#Port<5901.2653>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2104.0> 2011-07-14 00:19:51 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2104.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1947.0>] messages [] links [<5901.49.0>,<5901.1948.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1948.0> 2011-07-14 00:19:51 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2104.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2106.0> 2011-07-14 00:19:51 =============================================================================== ns_1@10.88.165.188:<5901.2106.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:51 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 81 times in the past 78.999357 secs (last seen 1.000341 secs ago PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2117.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2118.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2119.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2120.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2121.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2122.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2117.0> 2011-07-14 00:19:52 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2117.0> 2011-07-14 00:19:52 =============================================================================== ** Generic server <5901.2117.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.17870>}, init, {1310,617191,902663}, "default",#Port<5901.2654>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2117.0> 2011-07-14 00:19:52 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2117.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1947.0>] messages [] links [<5901.49.0>,<5901.1948.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1948.0> 2011-07-14 00:19:52 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2117.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:52 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 81 times in the past 79.659982 secs (last seen 1.001064 secs ago PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2126.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:52 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:52 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2127.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2128.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2129.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2130.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2131.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2126.0> 2011-07-14 00:19:52 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2126.0> 2011-07-14 00:19:52 =============================================================================== ** Generic server <5901.2126.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.17956>}, init, {1310,617192,406721}, "default",#Port<5901.2655>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2126.0> 2011-07-14 00:19:52 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2126.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1947.0>] messages [] links [<5901.49.0>,<5901.1948.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.1948.0> 2011-07-14 00:19:52 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2126.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2128.0> 2011-07-14 00:19:52 =============================================================================== ns_1@10.88.165.188:<5901.2128.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:52 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 82 times in the past 79.999186 secs (last seen 0.999829 secs ago PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2135.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2136.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2137.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2138.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2139.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2140.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2135.0> 2011-07-14 00:19:53 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2135.0> 2011-07-14 00:19:53 =============================================================================== ** Generic server <5901.2135.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.18034>}, init, {1310,617192,903107}, "default",#Port<5901.2656>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2135.0> 2011-07-14 00:19:53 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2135.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1947.0>] messages [] links [<5901.49.0>,<5901.1948.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.1948.0> 2011-07-14 00:19:53 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2135.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:53 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 82 times in the past 80.659072 secs (last seen 0.99909 secs ago PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2145.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2146.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2147.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:53 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:53 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2148.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2149.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2150.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2145.0> 2011-07-14 00:19:53 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2145.0> 2011-07-14 00:19:53 =============================================================================== ** Generic server <5901.2145.0> terminating ** Last message in was list_vbuckets_prevstate ** When Server state == {state,{interval,#Ref<5901.0.0.18157>}, init, {1310,617193,405666}, "default",#Port<5901.2658>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2145.0> 2011-07-14 00:19:53 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2145.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1947.0>] messages [{'$gen_call',{<5901.2147.0>,#Ref<5901.0.0.18258>},{stats,<<>>}}] links [<5901.49.0>,<5901.1948.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6671 SUPERVISOR REPORT <5901.1948.0> 2011-07-14 00:19:53 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2145.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker CRASH REPORT <5901.2160.0> 2011-07-14 00:19:53 =============================================================================== Crashing process initial_call {ns_janitor,cleanup,['Argument__1','Argument__2']} pid <5901.2160.0> registered_name [] error_info {exit,{{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, {gen_server,call, [{'ns_memcached-default','ns_1@10.88.165.188'}, list_vbuckets_prevstate,30000]}}, [{gen_server,call,3}, {ns_janitor,'-read_existing_map/4-lc$^0/1-0-',2}, {ns_janitor,read_existing_map,4}, {ns_janitor,cleanup,2}, {proc_lib,init_p_do_apply,3}]} ancestors [<5901.304.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<5901.52.0>] messages [] links [<5901.304.0>] dictionary [] trap_exit false status running heap_size 10946 stack_size 24 reductions 224 ERROR REPORT <5901.2147.0> 2011-07-14 00:19:53 =============================================================================== ns_1@10.88.165.188:<5901.2147.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:53 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 83 times in the past 80.99721 secs (last seen 0.998024 secs ago INFO REPORT <5901.304.0> 2011-07-14 00:19:53 =============================================================================== ns_1@10.88.165.188:<5901.304.0>:ns_orchestrator:178: Janitor run exited for bucket "default" with reason {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, [{'ns_memcached-default', 'ns_1@10.88.165.188'}, list_vbuckets_prevstate, 30000]}} PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2163.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2164.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2166.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2167.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2169.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.1948.0> 2011-07-14 00:19:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2170.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2163.0> 2011-07-14 00:19:54 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2163.0> 2011-07-14 00:19:54 =============================================================================== ** Generic server <5901.2163.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.18268>}, init, {1310,617193,900529}, "default",#Port<5901.2660>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2163.0> 2011-07-14 00:19:54 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2163.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.1947.0>] messages [] links [<5901.49.0>,<5901.1948.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.1948.0> 2011-07-14 00:19:54 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2163.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker SUPERVISOR REPORT <5901.1948.0> 2011-07-14 00:19:54 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext shutdown reason reached_max_restart_intensity pid <5901.2163.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:54 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 83 times in the past 81.655101 secs (last seen 0.996029 secs ago INFO REPORT <5901.1947.0> 2011-07-14 00:19:54 =============================================================================== ns_1@10.88.165.188:<5901.1947.0>:single_bucket_sup:24: per-bucket supervisor for "default" died with reason shutdown SUPERVISOR REPORT <5901.273.0> 2011-07-14 00:19:54 =============================================================================== Reporting supervisor {local,ns_bucket_sup} Child process errorContext child_terminated reason shutdown pid <5901.1947.0> name {per_bucket_sup,"default"} mfargs {single_bucket_sup,start_link,["default"]} restart_type permanent shutdown infinity child_type supervisor PROGRESS REPORT <5901.273.0> 2011-07-14 00:19:54 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<5901.2174.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2176.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2177.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2178.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2179.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2180.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2181.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2176.0> 2011-07-14 00:19:54 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2176.0> 2011-07-14 00:19:54 =============================================================================== ** Generic server <5901.2176.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.18366>}, init, {1310,617194,402039}, "default",#Port<5901.2661>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2176.0> 2011-07-14 00:19:54 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2176.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2174.0>] messages [] links [<5901.49.0>,<5901.2175.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2175.0> 2011-07-14 00:19:54 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2176.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2178.0> 2011-07-14 00:19:54 =============================================================================== ns_1@10.88.165.188:<5901.2178.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:54 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 84 times in the past 81.999232 secs (last seen 1.002022 secs ago PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2186.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:54 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:54 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2187.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2188.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2189.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2190.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2191.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2186.0> 2011-07-14 00:19:55 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2186.0> 2011-07-14 00:19:55 =============================================================================== ** Generic server <5901.2186.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.18466>}, init, {1310,617194,902627}, "default",#Port<5901.2662>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2186.0> 2011-07-14 00:19:55 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2186.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2174.0>] messages [] links [<5901.49.0>,<5901.2175.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2175.0> 2011-07-14 00:19:55 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2186.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:55 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 84 times in the past 82.656455 secs (last seen 1.001354 secs ago PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2195.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2196.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2197.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2198.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2199.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2200.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2195.0> 2011-07-14 00:19:55 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2195.0> 2011-07-14 00:19:55 =============================================================================== ** Generic server <5901.2195.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.18550>}, init, {1310,617195,402972}, "default",#Port<5901.2663>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2195.0> 2011-07-14 00:19:55 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2195.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2174.0>] messages [] links [<5901.49.0>,<5901.2175.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2175.0> 2011-07-14 00:19:55 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2195.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2197.0> 2011-07-14 00:19:55 =============================================================================== ns_1@10.88.165.188:<5901.2197.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:55 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 85 times in the past 82.999185 secs (last seen 0.999953 secs ago PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2207.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2208.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2209.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:55 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:55 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2210.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2211.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2212.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2207.0> 2011-07-14 00:19:56 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2207.0> 2011-07-14 00:19:56 =============================================================================== ** Generic server <5901.2207.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.18636>}, init, {1310,617195,902300}, "default",#Port<5901.2664>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2207.0> 2011-07-14 00:19:56 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2207.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2174.0>] messages [] links [<5901.49.0>,<5901.2175.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2175.0> 2011-07-14 00:19:56 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2207.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:56 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 85 times in the past 83.656366 secs (last seen 0.999911 secs ago PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2217.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2218.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2219.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2220.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2221.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2222.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2217.0> 2011-07-14 00:19:56 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2217.0> 2011-07-14 00:19:56 =============================================================================== ** Generic server <5901.2217.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.18762>}, init, {1310,617196,402894}, "default",#Port<5901.2666>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2217.0> 2011-07-14 00:19:56 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2217.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2174.0>] messages [] links [<5901.49.0>,<5901.2175.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2175.0> 2011-07-14 00:19:56 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2217.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2219.0> 2011-07-14 00:19:56 =============================================================================== ns_1@10.88.165.188:<5901.2219.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:56 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 86 times in the past 83.999145 secs (last seen 0.99996 secs ago PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2226.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2227.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2228.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2229.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2230.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2231.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2226.0> 2011-07-14 00:19:57 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2226.0> 2011-07-14 00:19:57 =============================================================================== ** Generic server <5901.2226.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.18840>}, init, {1310,617196,902251}, "default",#Port<5901.2667>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2226.0> 2011-07-14 00:19:57 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2226.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2174.0>] messages [] links [<5901.49.0>,<5901.2175.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2175.0> 2011-07-14 00:19:57 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2226.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:57 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 86 times in the past 84.656285 secs (last seen 0.999919 secs ago PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2235.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2236.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:19:57 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:57 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2237.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2238.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2239.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2240.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2235.0> 2011-07-14 00:19:57 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2235.0> 2011-07-14 00:19:57 =============================================================================== ** Generic server <5901.2235.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.18926>}, init, {1310,617197,402838}, "default",#Port<5901.2668>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2235.0> 2011-07-14 00:19:57 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2235.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2174.0>] messages [] links [<5901.49.0>,<5901.2175.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2175.0> 2011-07-14 00:19:57 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2235.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2237.0> 2011-07-14 00:19:57 =============================================================================== ns_1@10.88.165.188:<5901.2237.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:57 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 87 times in the past 84.999238 secs (last seen 1.000093 secs ago PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2247.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2248.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2249.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2250.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2251.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2252.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2247.0> 2011-07-14 00:19:58 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2247.0> 2011-07-14 00:19:58 =============================================================================== ** Generic server <5901.2247.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.19007>}, init, {1310,617197,902474}, "default",#Port<5901.2669>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2247.0> 2011-07-14 00:19:58 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2247.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2174.0>] messages [] links [<5901.49.0>,<5901.2175.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2175.0> 2011-07-14 00:19:58 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2247.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:58 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 87 times in the past 85.656237 secs (last seen 0.999952 secs ago PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2257.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2258.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2259.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2260.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2261.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2262.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2257.0> 2011-07-14 00:19:58 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2257.0> 2011-07-14 00:19:58 =============================================================================== ** Generic server <5901.2257.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.19116>}, init, {1310,617198,402668}, "default",#Port<5901.2670>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2257.0> 2011-07-14 00:19:58 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2257.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2174.0>] messages [] links [<5901.49.0>,<5901.2175.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6665 SUPERVISOR REPORT <5901.2175.0> 2011-07-14 00:19:58 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2257.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2259.0> 2011-07-14 00:19:58 =============================================================================== ns_1@10.88.165.188:<5901.2259.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:58 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 88 times in the past 85.999211 secs (last seen 0.999973 secs ago INFO REPORT <5901.269.0> 2011-07-14 00:19:58 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:19:58 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2270.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2272.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2273.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2274.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2275.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2276.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2270.0> 2011-07-14 00:19:59 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2270.0> 2011-07-14 00:19:59 =============================================================================== ** Generic server <5901.2270.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.19289>}, init, {1310,617198,902789}, "default",#Port<5901.2672>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2270.0> 2011-07-14 00:19:59 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2270.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2174.0>] messages [] links [<5901.49.0>,<5901.2175.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.2175.0> 2011-07-14 00:19:59 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2270.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:19:59 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 88 times in the past 86.656348 secs (last seen 1.000111 secs ago PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2282.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2283.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2284.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2285.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2286.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2287.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2282.0> 2011-07-14 00:19:59 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2282.0> 2011-07-14 00:19:59 =============================================================================== ** Generic server <5901.2282.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.19416>}, init, {1310,617199,402913}, "default",#Port<5901.2674>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2282.0> 2011-07-14 00:19:59 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2282.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2174.0>] messages [] links [<5901.49.0>,<5901.2175.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2175.0> 2011-07-14 00:19:59 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2282.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2284.0> 2011-07-14 00:19:59 =============================================================================== ns_1@10.88.165.188:<5901.2284.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:19:59 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 89 times in the past 86.99934 secs (last seen 1.000129 secs ago PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2294.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2295.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2296.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2297.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2298.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:19:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2299.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2294.0> 2011-07-14 00:20:00 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2294.0> 2011-07-14 00:20:00 =============================================================================== ** Generic server <5901.2294.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.19503>}, init, {1310,617199,902933}, "default",#Port<5901.2675>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2294.0> 2011-07-14 00:20:00 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2294.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2174.0>] messages [] links [<5901.49.0>,<5901.2175.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2175.0> 2011-07-14 00:20:00 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2294.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:00 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 89 times in the past 87.65647 secs (last seen 1.000122 secs ago PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2303.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2304.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:00 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:00 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2305.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2306.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2307.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2308.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.239.0> 2011-07-14 00:20:00 =============================================================================== ns_1@10.88.165.188:<5901.239.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.2303.0> 2011-07-14 00:20:00 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2303.0> 2011-07-14 00:20:00 =============================================================================== ** Generic server <5901.2303.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.19590>}, init, {1310,617200,402975}, "default",#Port<5901.2676>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2303.0> 2011-07-14 00:20:00 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2303.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2174.0>] messages [] links [<5901.49.0>,<5901.2175.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2175.0> 2011-07-14 00:20:00 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2303.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2305.0> 2011-07-14 00:20:00 =============================================================================== ns_1@10.88.165.188:<5901.2305.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:00 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 90 times in the past 87.999616 secs (last seen 1.000276 secs ago PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2314.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2315.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2316.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2317.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2318.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2319.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2314.0> 2011-07-14 00:20:01 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2314.0> 2011-07-14 00:20:01 =============================================================================== ** Generic server <5901.2314.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.19675>}, init, {1310,617200,902655}, "default",#Port<5901.2678>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2314.0> 2011-07-14 00:20:01 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2314.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2174.0>] messages [] links [<5901.49.0>,<5901.2175.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2175.0> 2011-07-14 00:20:01 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2314.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:01 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 90 times in the past 88.659492 secs (last seen 1.003022 secs ago PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2324.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2325.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2326.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2327.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2328.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2329.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2324.0> 2011-07-14 00:20:01 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2324.0> 2011-07-14 00:20:01 =============================================================================== ** Generic server <5901.2324.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.19783>}, init, {1310,617201,406746}, "default",#Port<5901.2679>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2324.0> 2011-07-14 00:20:01 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2324.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2174.0>] messages [] links [<5901.49.0>,<5901.2175.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2175.0> 2011-07-14 00:20:01 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2324.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2326.0> 2011-07-14 00:20:01 =============================================================================== ns_1@10.88.165.188:<5901.2326.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:01 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 91 times in the past 88.999707 secs (last seen 1.000091 secs ago PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2336.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:01 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:01 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2337.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2338.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2339.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2340.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2341.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2336.0> 2011-07-14 00:20:02 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2336.0> 2011-07-14 00:20:02 =============================================================================== ** Generic server <5901.2336.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.19864>}, init, {1310,617201,903127}, "default",#Port<5901.2680>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2336.0> 2011-07-14 00:20:02 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2336.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2174.0>] messages [] links [<5901.49.0>,<5901.2175.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2175.0> 2011-07-14 00:20:02 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2336.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:02 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 91 times in the past 89.659556 secs (last seen 1.000064 secs ago PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2346.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2347.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2348.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2349.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2350.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2351.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2346.0> 2011-07-14 00:20:02 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2346.0> 2011-07-14 00:20:02 =============================================================================== ** Generic server <5901.2346.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.19996>}, init, {1310,617202,406333}, "default",#Port<5901.2682>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2346.0> 2011-07-14 00:20:02 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2346.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2174.0>] messages [] links [<5901.49.0>,<5901.2175.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2175.0> 2011-07-14 00:20:02 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2346.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2348.0> 2011-07-14 00:20:02 =============================================================================== ns_1@10.88.165.188:<5901.2348.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:02 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 92 times in the past 89.99976 secs (last seen 1.000053 secs ago PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2355.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2356.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2357.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:02 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:02 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2358.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2359.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2360.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2355.0> 2011-07-14 00:20:03 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2355.0> 2011-07-14 00:20:03 =============================================================================== ** Generic server <5901.2355.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.20077>}, init, {1310,617202,902862}, "default",#Port<5901.2683>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2355.0> 2011-07-14 00:20:03 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2355.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2174.0>] messages [] links [<5901.49.0>,<5901.2175.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2175.0> 2011-07-14 00:20:03 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2355.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:03 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 92 times in the past 90.659657 secs (last seen 1.000101 secs ago PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2364.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2365.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2366.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2367.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2368.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2369.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2364.0> 2011-07-14 00:20:03 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2364.0> 2011-07-14 00:20:03 =============================================================================== ** Generic server <5901.2364.0> terminating ** Last message in was list_vbuckets_prevstate ** When Server state == {state,{interval,#Ref<5901.0.0.20159>}, init, {1310,617203,406260}, "default",#Port<5901.2684>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2364.0> 2011-07-14 00:20:03 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2364.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2174.0>] messages [{'$gen_call',{<5901.2366.0>,#Ref<5901.0.0.20266>},{stats,<<>>}}] links [<5901.49.0>,<5901.2175.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6671 SUPERVISOR REPORT <5901.2175.0> 2011-07-14 00:20:03 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2364.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker CRASH REPORT <5901.2379.0> 2011-07-14 00:20:03 =============================================================================== Crashing process initial_call {ns_janitor,cleanup,['Argument__1','Argument__2']} pid <5901.2379.0> registered_name [] error_info {exit,{{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, {gen_server,call, [{'ns_memcached-default','ns_1@10.88.165.188'}, list_vbuckets_prevstate,30000]}}, [{gen_server,call,3}, {ns_janitor,'-read_existing_map/4-lc$^0/1-0-',2}, {ns_janitor,read_existing_map,4}, {ns_janitor,cleanup,2}, {proc_lib,init_p_do_apply,3}]} ancestors [<5901.304.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<5901.52.0>] messages [] links [<5901.304.0>] dictionary [] trap_exit false status running heap_size 10946 stack_size 24 reductions 224 ERROR REPORT <5901.2366.0> 2011-07-14 00:20:03 =============================================================================== ns_1@10.88.165.188:<5901.2366.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:03 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 93 times in the past 90.997666 secs (last seen 0.997906 secs ago INFO REPORT <5901.304.0> 2011-07-14 00:20:03 =============================================================================== ns_1@10.88.165.188:<5901.304.0>:ns_orchestrator:178: Janitor run exited for bucket "default" with reason {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, [{'ns_memcached-default', 'ns_1@10.88.165.188'}, list_vbuckets_prevstate, 30000]}} PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2382.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2383.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2385.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2386.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2387.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2388.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2382.0> 2011-07-14 00:20:04 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2382.0> 2011-07-14 00:20:04 =============================================================================== ** Generic server <5901.2382.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.20276>}, init, {1310,617203,900923}, "default",#Port<5901.2686>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2382.0> 2011-07-14 00:20:04 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2382.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2174.0>] messages [] links [<5901.49.0>,<5901.2175.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.2175.0> 2011-07-14 00:20:04 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2382.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:04 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 93 times in the past 91.655512 secs (last seen 0.995855 secs ago PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2394.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2395.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2396.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2397.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2398.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2175.0> 2011-07-14 00:20:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2399.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:04 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:04 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.2394.0> 2011-07-14 00:20:04 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2394.0> 2011-07-14 00:20:04 =============================================================================== ** Generic server <5901.2394.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.20392>}, init, {1310,617204,402058}, "default",#Port<5901.2687>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2394.0> 2011-07-14 00:20:04 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2394.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2174.0>] messages [] links [<5901.49.0>,<5901.2175.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2175.0> 2011-07-14 00:20:04 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2394.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker SUPERVISOR REPORT <5901.2175.0> 2011-07-14 00:20:04 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext shutdown reason reached_max_restart_intensity pid <5901.2394.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2396.0> 2011-07-14 00:20:04 =============================================================================== ns_1@10.88.165.188:<5901.2396.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:04 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 94 times in the past 91.999675 secs (last seen 1.002009 secs ago INFO REPORT <5901.2174.0> 2011-07-14 00:20:04 =============================================================================== ns_1@10.88.165.188:<5901.2174.0>:single_bucket_sup:24: per-bucket supervisor for "default" died with reason shutdown SUPERVISOR REPORT <5901.273.0> 2011-07-14 00:20:04 =============================================================================== Reporting supervisor {local,ns_bucket_sup} Child process errorContext child_terminated reason shutdown pid <5901.2174.0> name {per_bucket_sup,"default"} mfargs {single_bucket_sup,start_link,["default"]} restart_type permanent shutdown infinity child_type supervisor PROGRESS REPORT <5901.273.0> 2011-07-14 00:20:04 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<5901.2403.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2405.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2406.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2407.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2408.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2409.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2410.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2405.0> 2011-07-14 00:20:05 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2405.0> 2011-07-14 00:20:05 =============================================================================== ** Generic server <5901.2405.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.20468>}, init, {1310,617204,903259}, "default",#Port<5901.2688>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2405.0> 2011-07-14 00:20:05 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2405.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2403.0>] messages [] links [<5901.49.0>,<5901.2404.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2404.0> 2011-07-14 00:20:05 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2405.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:05 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 94 times in the past 92.65697 secs (last seen 1.001458 secs ago PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2415.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2416.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2417.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2418.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2419.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2420.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2415.0> 2011-07-14 00:20:05 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2415.0> 2011-07-14 00:20:05 =============================================================================== ** Generic server <5901.2415.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.20593>}, init, {1310,617205,403526}, "default",#Port<5901.2690>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2415.0> 2011-07-14 00:20:05 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2415.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2403.0>] messages [] links [<5901.49.0>,<5901.2404.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2404.0> 2011-07-14 00:20:05 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2415.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2417.0> 2011-07-14 00:20:05 =============================================================================== ns_1@10.88.165.188:<5901.2417.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:05 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 95 times in the past 92.999775 secs (last seen 1.0001 secs ago PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2427.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2428.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2429.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:05 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:05 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2430.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2431.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2432.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2427.0> 2011-07-14 00:20:06 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2427.0> 2011-07-14 00:20:06 =============================================================================== ** Generic server <5901.2427.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.20674>}, init, {1310,617205,902805}, "default",#Port<5901.2691>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2427.0> 2011-07-14 00:20:06 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2427.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2403.0>] messages [] links [<5901.49.0>,<5901.2404.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2404.0> 2011-07-14 00:20:06 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2427.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:06 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 95 times in the past 93.65696 secs (last seen 0.99999 secs ago PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2436.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2437.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2438.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2439.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2440.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2441.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2436.0> 2011-07-14 00:20:06 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2436.0> 2011-07-14 00:20:06 =============================================================================== ** Generic server <5901.2436.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.20760>}, init, {1310,617206,403494}, "default",#Port<5901.2692>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2436.0> 2011-07-14 00:20:06 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2436.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2403.0>] messages [] links [<5901.49.0>,<5901.2404.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2404.0> 2011-07-14 00:20:06 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2436.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2438.0> 2011-07-14 00:20:06 =============================================================================== ns_1@10.88.165.188:<5901.2438.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:06 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 96 times in the past 94.000018 secs (last seen 1.000243 secs ago PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2445.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2446.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2447.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2448.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2449.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2450.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2445.0> 2011-07-14 00:20:07 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2445.0> 2011-07-14 00:20:07 =============================================================================== ** Generic server <5901.2445.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.20842>}, init, {1310,617206,903928}, "default",#Port<5901.2693>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2445.0> 2011-07-14 00:20:07 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2445.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2403.0>] messages [] links [<5901.49.0>,<5901.2404.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2404.0> 2011-07-14 00:20:07 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2445.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:07 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 96 times in the past 94.657041 secs (last seen 1.000081 secs ago PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2455.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2456.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2457.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:07 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:07 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2458.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2459.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2460.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2455.0> 2011-07-14 00:20:07 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2455.0> 2011-07-14 00:20:07 =============================================================================== ** Generic server <5901.2455.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.20944>}, init, {1310,617207,403572}, "default",#Port<5901.2694>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2455.0> 2011-07-14 00:20:07 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2455.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2403.0>] messages [] links [<5901.49.0>,<5901.2404.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2404.0> 2011-07-14 00:20:07 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2455.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2457.0> 2011-07-14 00:20:07 =============================================================================== ns_1@10.88.165.188:<5901.2457.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:07 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 97 times in the past 94.999889 secs (last seen 0.999871 secs ago PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2467.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2468.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2469.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2470.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2471.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2472.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2467.0> 2011-07-14 00:20:08 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2467.0> 2011-07-14 00:20:08 =============================================================================== ** Generic server <5901.2467.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.21030>}, init, {1310,617207,903073}, "default",#Port<5901.2695>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2467.0> 2011-07-14 00:20:08 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2467.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2403.0>] messages [] links [<5901.49.0>,<5901.2404.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2404.0> 2011-07-14 00:20:08 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2467.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:08 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 97 times in the past 95.659718 secs (last seen 1.002677 secs ago PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2477.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2478.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2479.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2480.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2481.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2482.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2477.0> 2011-07-14 00:20:08 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2477.0> 2011-07-14 00:20:08 =============================================================================== ** Generic server <5901.2477.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.21158>}, init, {1310,617208,406229}, "default",#Port<5901.2697>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2477.0> 2011-07-14 00:20:08 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2477.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2403.0>] messages [] links [<5901.49.0>,<5901.2404.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6665 SUPERVISOR REPORT <5901.2404.0> 2011-07-14 00:20:08 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2477.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2479.0> 2011-07-14 00:20:08 =============================================================================== ns_1@10.88.165.188:<5901.2479.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:08 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 98 times in the past 95.999903 secs (last seen 1.000014 secs ago PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2489.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:08 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:08 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2490.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2492.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2493.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2495.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2496.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2489.0> 2011-07-14 00:20:09 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2489.0> 2011-07-14 00:20:09 =============================================================================== ** Generic server <5901.2489.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.21251>}, init, {1310,617208,902867}, "default",#Port<5901.2700>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2489.0> 2011-07-14 00:20:09 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2489.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2403.0>] messages [] links [<5901.49.0>,<5901.2404.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.2404.0> 2011-07-14 00:20:09 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2489.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:09 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 98 times in the past 96.659952 secs (last seen 1.000234 secs ago PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2500.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2501.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2502.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2503.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2504.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2505.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2500.0> 2011-07-14 00:20:09 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2500.0> 2011-07-14 00:20:09 =============================================================================== ** Generic server <5901.2500.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.21348>}, init, {1310,617209,406619}, "default",#Port<5901.2701>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2500.0> 2011-07-14 00:20:09 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2500.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2403.0>] messages [] links [<5901.49.0>,<5901.2404.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2404.0> 2011-07-14 00:20:09 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2500.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2502.0> 2011-07-14 00:20:09 =============================================================================== ns_1@10.88.165.188:<5901.2502.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:09 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 99 times in the past 97.000011 secs (last seen 1.000108 secs ago PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2512.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2513.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2514.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:09 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:09 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2515.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2516.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2517.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2512.0> 2011-07-14 00:20:10 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2512.0> 2011-07-14 00:20:10 =============================================================================== ** Generic server <5901.2512.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.21429>}, init, {1310,617209,903124}, "default",#Port<5901.2702>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2512.0> 2011-07-14 00:20:10 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2512.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2403.0>] messages [] links [<5901.49.0>,<5901.2404.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2404.0> 2011-07-14 00:20:10 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2512.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:10 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 99 times in the past 97.660011 secs (last seen 1.000059 secs ago PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2521.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2522.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2523.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2524.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2525.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2526.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2521.0> 2011-07-14 00:20:10 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2521.0> 2011-07-14 00:20:10 =============================================================================== ** Generic server <5901.2521.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.21516>}, init, {1310,617210,406670}, "default",#Port<5901.2703>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2521.0> 2011-07-14 00:20:10 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2521.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2403.0>] messages [] links [<5901.49.0>,<5901.2404.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2404.0> 2011-07-14 00:20:10 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2521.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2523.0> 2011-07-14 00:20:10 =============================================================================== ns_1@10.88.165.188:<5901.2523.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:10 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 100 times in the past 98.000238 secs (last seen 1.000227 secs ago PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2531.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2532.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2533.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2534.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2535.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2536.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2531.0> 2011-07-14 00:20:11 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2531.0> 2011-07-14 00:20:11 =============================================================================== ** Generic server <5901.2531.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.21618>}, init, {1310,617210,903454}, "default",#Port<5901.2704>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2531.0> 2011-07-14 00:20:11 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2531.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2403.0>] messages [] links [<5901.49.0>,<5901.2404.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2404.0> 2011-07-14 00:20:11 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2531.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:11 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 100 times in the past 98.657415 secs (last seen 0.997404 secs ago PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2541.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2542.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2543.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2544.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2545.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2546.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:11 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:11 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.2541.0> 2011-07-14 00:20:11 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2541.0> 2011-07-14 00:20:11 =============================================================================== ** Generic server <5901.2541.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.21736>}, init, {1310,617211,403992}, "default",#Port<5901.2706>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2541.0> 2011-07-14 00:20:11 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2541.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2403.0>] messages [] links [<5901.49.0>,<5901.2404.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2404.0> 2011-07-14 00:20:11 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2541.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2543.0> 2011-07-14 00:20:11 =============================================================================== ns_1@10.88.165.188:<5901.2543.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:11 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 101 times in the past 99.000152 secs (last seen 0.999914 secs ago PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2553.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2554.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2555.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2556.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2557.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2558.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2553.0> 2011-07-14 00:20:12 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2553.0> 2011-07-14 00:20:12 =============================================================================== ** Generic server <5901.2553.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.21821>}, init, {1310,617211,903994}, "default",#Port<5901.2707>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2553.0> 2011-07-14 00:20:12 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2553.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2403.0>] messages [] links [<5901.49.0>,<5901.2404.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2404.0> 2011-07-14 00:20:12 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2553.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:12 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 101 times in the past 99.657235 secs (last seen 0.99982 secs ago PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2562.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2563.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2564.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2565.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2566.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2567.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2562.0> 2011-07-14 00:20:12 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2562.0> 2011-07-14 00:20:12 =============================================================================== ** Generic server <5901.2562.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.21910>}, init, {1310,617212,403729}, "default",#Port<5901.2708>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2562.0> 2011-07-14 00:20:12 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2562.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2403.0>] messages [] links [<5901.49.0>,<5901.2404.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2404.0> 2011-07-14 00:20:12 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2562.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2564.0> 2011-07-14 00:20:12 =============================================================================== ns_1@10.88.165.188:<5901.2564.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:12 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 102 times in the past 99.997471 secs (last seen 0.997319 secs ago PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2571.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2572.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2573.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2574.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2575.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2576.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:12 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:12 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.2571.0> 2011-07-14 00:20:13 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2571.0> 2011-07-14 00:20:13 =============================================================================== ** Generic server <5901.2571.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.21996>}, init, {1310,617212,900591}, "default",#Port<5901.2710>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2571.0> 2011-07-14 00:20:13 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2571.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2403.0>] messages [] links [<5901.49.0>,<5901.2404.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2404.0> 2011-07-14 00:20:13 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2571.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:13 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 102 times in the past 100.65629 secs (last seen 0.999055 secs ago PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2580.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2581.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2582.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2583.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2584.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2585.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.256.0> 2011-07-14 00:20:13 =============================================================================== ns_1@10.88.165.188:<5901.256.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.88.165.188', [{last_heard,{1310,617213,599918}}, {active_buckets,["default"]}, {ready_buckets,[]}, {replication,[{"default",1.0}]}, {memory, [{total,28586296}, {processes,6874984}, {processes_used,6867000}, {system,21711312}, {atom,814593}, {atom_used,805810}, {binary,489816}, {code,7689836}, {ets,3415768}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,121}, {memory_data,{644874240,582463488,{<5901.80.0>,601112}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 68708 kB\nBuffers: 22104 kB\nCached: 365624 kB\nSwapCached: 0 kB\nActive: 215004 kB\nInactive: 269504 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 68708 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 172 kB\nWriteback: 0 kB\nAnonPages: 96768 kB\nMapped: 11860 kB\nSlab: 36696 kB\nPageTables: 3812 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 545028 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,374398976}, {buffered_memory,22634496}, {free_memory,70356992}, {total_memory,644874240}]}, {statistics, [{wall_clock,{106634,1}}, {context_switches,{57284,0}}, {garbage_collection,{12196,25255010,0}}, {io,{{input,15705091},{output,7170208}}}, {reductions,{6885364,864706}}, {run_queue,0}, {runtime,{550,10}}]}]}, {'ns_1@10.88.23.62', [{last_heard,{1310,617213,609883}}, {active_buckets,["default"]}, {ready_buckets,[]}, {replication,[{"default",1.0}]}, {memory, [{total,30016096}, {processes,7803144}, {processes_used,7776840}, {system,22212952}, {atom,820249}, {atom_used,814132}, {binary,917680}, {code,7741308}, {ets,3430472}]}, {system_stats, [{cpu_utilization_rate,1.9801980198019802}, {swap_total,0}, {swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,125}, {memory_data,{644874240,587669504,{<5990.256.0>,830584}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 55864 kB\nBuffers: 21644 kB\nCached: 367524 kB\nSwapCached: 0 kB\nActive: 227364 kB\nInactive: 268620 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 55864 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 932 kB\nWriteback: 0 kB\nAnonPages: 106824 kB\nMapped: 12316 kB\nSlab: 37308 kB\nPageTables: 4116 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 554844 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,376344576}, {buffered_memory,22163456}, {free_memory,57204736}, {total_memory,644874240}]}, {statistics, [{wall_clock,{121502,0}}, {context_switches,{70849,0}}, {garbage_collection,{14441,41209612,0}}, {io,{{input,16505423},{output,7528245}}}, {reductions,{8689931,935016}}, {run_queue,0}, {runtime,{890,30}}]}]}] INFO REPORT <5901.2580.0> 2011-07-14 00:20:13 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2580.0> 2011-07-14 00:20:13 =============================================================================== ** Generic server <5901.2580.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.22082>}, init, {1310,617213,403092}, "default",#Port<5901.2711>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2580.0> 2011-07-14 00:20:13 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2580.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2403.0>] messages [] links [<5901.49.0>,<5901.2404.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6671 SUPERVISOR REPORT <5901.2404.0> 2011-07-14 00:20:13 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2580.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2582.0> 2011-07-14 00:20:13 =============================================================================== ns_1@10.88.165.188:<5901.2582.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:13 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 103 times in the past 100.997227 secs (last seen 0.999756 secs ago PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2601.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2602.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2604.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2605.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2606.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2607.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2601.0> 2011-07-14 00:20:13 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2601.0> 2011-07-14 00:20:13 =============================================================================== ** Generic server <5901.2601.0> terminating ** Last message in was list_vbuckets_prevstate ** When Server state == {state,{interval,#Ref<5901.0.0.22294>}, init, {1310,617213,900147}, "default",#Port<5901.2714>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2601.0> 2011-07-14 00:20:13 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2601.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2403.0>] messages [] links [<5901.49.0>,<5901.2404.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6701 SUPERVISOR REPORT <5901.2404.0> 2011-07-14 00:20:13 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2601.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker CRASH REPORT <5901.2594.0> 2011-07-14 00:20:13 =============================================================================== Crashing process initial_call {ns_janitor,cleanup,['Argument__1','Argument__2']} pid <5901.2594.0> registered_name [] error_info {exit,{{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, {gen_server,call, [{'ns_memcached-default','ns_1@10.88.165.188'}, list_vbuckets_prevstate,30000]}}, [{gen_server,call,3}, {ns_janitor,'-read_existing_map/4-lc$^0/1-0-',2}, {ns_janitor,read_existing_map,4}, {ns_janitor,cleanup,2}, {proc_lib,init_p_do_apply,3}]} ancestors [<5901.304.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<5901.52.0>] messages [] links [<5901.304.0>] dictionary [] trap_exit false status running heap_size 10946 stack_size 24 reductions 224 INFO REPORT <5901.227.0> 2011-07-14 00:20:13 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 104 times in the past 101.006818 secs (last seen 0.009591 secs ago INFO REPORT <5901.304.0> 2011-07-14 00:20:13 =============================================================================== ns_1@10.88.165.188:<5901.304.0>:ns_orchestrator:178: Janitor run exited for bucket "default" with reason {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, [{'ns_memcached-default', 'ns_1@10.88.165.188'}, list_vbuckets_prevstate, 30000]}} PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2609.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2610.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2611.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2612.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2613.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2614.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:14 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:14 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.2609.0> 2011-07-14 00:20:14 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2609.0> 2011-07-14 00:20:14 =============================================================================== ** Generic server <5901.2609.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.22369>}, init, {1310,617213,908812}, "default",#Port<5901.2715>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2609.0> 2011-07-14 00:20:14 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2609.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2403.0>] messages [] links [<5901.49.0>,<5901.2404.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2404.0> 2011-07-14 00:20:14 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2609.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:14 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 103 times in the past 101.664613 secs (last seen 1.008323 secs ago PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2619.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2620.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2621.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2622.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2623.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2404.0> 2011-07-14 00:20:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2624.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2619.0> 2011-07-14 00:20:14 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2619.0> 2011-07-14 00:20:14 =============================================================================== ** Generic server <5901.2619.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.22497>}, init, {1310,617214,411263}, "default",#Port<5901.2717>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2619.0> 2011-07-14 00:20:14 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2619.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2403.0>] messages [] links [<5901.49.0>,<5901.2404.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2404.0> 2011-07-14 00:20:14 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2619.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker SUPERVISOR REPORT <5901.2404.0> 2011-07-14 00:20:14 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext shutdown reason reached_max_restart_intensity pid <5901.2619.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2621.0> 2011-07-14 00:20:14 =============================================================================== ns_1@10.88.165.188:<5901.2621.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:14 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 105 times in the past 101.997533 secs (last seen 0.990715 secs ago INFO REPORT <5901.2403.0> 2011-07-14 00:20:14 =============================================================================== ns_1@10.88.165.188:<5901.2403.0>:single_bucket_sup:24: per-bucket supervisor for "default" died with reason shutdown SUPERVISOR REPORT <5901.273.0> 2011-07-14 00:20:14 =============================================================================== Reporting supervisor {local,ns_bucket_sup} Child process errorContext child_terminated reason shutdown pid <5901.2403.0> name {per_bucket_sup,"default"} mfargs {single_bucket_sup,start_link,["default"]} restart_type permanent shutdown infinity child_type supervisor PROGRESS REPORT <5901.273.0> 2011-07-14 00:20:14 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<5901.2628.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2630.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2631.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2632.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2633.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2634.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2635.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2630.0> 2011-07-14 00:20:15 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2630.0> 2011-07-14 00:20:15 =============================================================================== ** Generic server <5901.2630.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.22580>}, init, {1310,617214,900872}, "default",#Port<5901.2718>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2630.0> 2011-07-14 00:20:15 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2630.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2628.0>] messages [] links [<5901.49.0>,<5901.2629.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2629.0> 2011-07-14 00:20:15 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2630.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:15 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 104 times in the past 102.656376 secs (last seen 0.991763 secs ago PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2639.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2640.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2641.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2642.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2643.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2644.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:15 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:15 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.2639.0> 2011-07-14 00:20:15 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2639.0> 2011-07-14 00:20:15 =============================================================================== ** Generic server <5901.2639.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.22662>}, init, {1310,617215,402959}, "default",#Port<5901.2719>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2639.0> 2011-07-14 00:20:15 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2639.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2628.0>] messages [] links [<5901.49.0>,<5901.2629.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2629.0> 2011-07-14 00:20:15 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2639.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2641.0> 2011-07-14 00:20:15 =============================================================================== ns_1@10.88.165.188:<5901.2641.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:15 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 106 times in the past 102.997351 secs (last seen 0.999818 secs ago PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2651.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2652.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2653.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2654.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2655.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2656.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2651.0> 2011-07-14 00:20:16 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2651.0> 2011-07-14 00:20:16 =============================================================================== ** Generic server <5901.2651.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.22747>}, init, {1310,617215,900353}, "default",#Port<5901.2720>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2651.0> 2011-07-14 00:20:16 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2651.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2628.0>] messages [] links [<5901.49.0>,<5901.2629.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2629.0> 2011-07-14 00:20:16 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2651.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:16 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 105 times in the past 103.656251 secs (last seen 0.999875 secs ago PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2660.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2661.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2662.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2663.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2664.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2665.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2660.0> 2011-07-14 00:20:16 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2660.0> 2011-07-14 00:20:16 =============================================================================== ** Generic server <5901.2660.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.22835>}, init, {1310,617216,402763}, "default",#Port<5901.2721>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2660.0> 2011-07-14 00:20:16 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2660.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2628.0>] messages [] links [<5901.49.0>,<5901.2629.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2629.0> 2011-07-14 00:20:16 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2660.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2662.0> 2011-07-14 00:20:16 =============================================================================== ns_1@10.88.165.188:<5901.2662.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:16 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 107 times in the past 103.997315 secs (last seen 0.999964 secs ago PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2670.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2671.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2672.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:16 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:16 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2673.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2674.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2675.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2670.0> 2011-07-14 00:20:17 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2670.0> 2011-07-14 00:20:17 =============================================================================== ** Generic server <5901.2670.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.22931>}, init, {1310,617216,900231}, "default",#Port<5901.2722>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2670.0> 2011-07-14 00:20:17 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2670.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2628.0>] messages [] links [<5901.49.0>,<5901.2629.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2629.0> 2011-07-14 00:20:17 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2670.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:17 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 106 times in the past 104.656476 secs (last seen 1.000225 secs ago PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2680.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2681.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2682.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2683.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2684.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2685.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2680.0> 2011-07-14 00:20:17 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2680.0> 2011-07-14 00:20:17 =============================================================================== ** Generic server <5901.2680.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.23054>}, init, {1310,617217,403119}, "default",#Port<5901.2724>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2680.0> 2011-07-14 00:20:17 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2680.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2628.0>] messages [] links [<5901.49.0>,<5901.2629.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2629.0> 2011-07-14 00:20:17 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2680.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2682.0> 2011-07-14 00:20:17 =============================================================================== ns_1@10.88.165.188:<5901.2682.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:17 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 108 times in the past 104.997337 secs (last seen 1.000022 secs ago PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2692.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2693.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2694.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2695.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2696.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2697.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2692.0> 2011-07-14 00:20:18 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2692.0> 2011-07-14 00:20:18 =============================================================================== ** Generic server <5901.2692.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.23137>}, init, {1310,617217,900370}, "default",#Port<5901.2725>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2692.0> 2011-07-14 00:20:18 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2692.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2628.0>] messages [] links [<5901.49.0>,<5901.2629.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2629.0> 2011-07-14 00:20:18 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2692.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:18 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 107 times in the past 105.656517 secs (last seen 1.000041 secs ago PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2701.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:18 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:18 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2702.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2703.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2704.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2705.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2706.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2701.0> 2011-07-14 00:20:18 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2701.0> 2011-07-14 00:20:18 =============================================================================== ** Generic server <5901.2701.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.23223>}, init, {1310,617218,403109}, "default",#Port<5901.2726>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2701.0> 2011-07-14 00:20:18 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2701.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2628.0>] messages [] links [<5901.49.0>,<5901.2629.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6665 SUPERVISOR REPORT <5901.2629.0> 2011-07-14 00:20:18 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2701.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2703.0> 2011-07-14 00:20:18 =============================================================================== ns_1@10.88.165.188:<5901.2703.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:18 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 109 times in the past 105.997322 secs (last seen 0.999985 secs ago PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2713.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2714.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2716.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2717.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2718.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2719.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2713.0> 2011-07-14 00:20:19 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2713.0> 2011-07-14 00:20:19 =============================================================================== ** Generic server <5901.2713.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.23321>}, init, {1310,617218,901005}, "default",#Port<5901.2727>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2713.0> 2011-07-14 00:20:19 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2713.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2628.0>] messages [] links [<5901.49.0>,<5901.2629.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.2629.0> 2011-07-14 00:20:19 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2713.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:19 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 108 times in the past 106.65452 secs (last seen 0.998003 secs ago PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2724.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2725.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2726.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2727.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2728.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2729.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:19 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:19 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.2724.0> 2011-07-14 00:20:19 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2724.0> 2011-07-14 00:20:19 =============================================================================== ** Generic server <5901.2724.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.23412>}, init, {1310,617219,401061}, "default",#Port<5901.2728>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2724.0> 2011-07-14 00:20:19 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2724.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2628.0>] messages [] links [<5901.49.0>,<5901.2629.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2629.0> 2011-07-14 00:20:19 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2724.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2726.0> 2011-07-14 00:20:19 =============================================================================== ns_1@10.88.165.188:<5901.2726.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:19 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 110 times in the past 106.997558 secs (last seen 1.000236 secs ago PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2736.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2737.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2738.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2739.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2740.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2741.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2736.0> 2011-07-14 00:20:20 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2736.0> 2011-07-14 00:20:20 =============================================================================== ** Generic server <5901.2736.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.23498>}, init, {1310,617219,900692}, "default",#Port<5901.2729>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2736.0> 2011-07-14 00:20:20 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2736.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2628.0>] messages [] links [<5901.49.0>,<5901.2629.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2629.0> 2011-07-14 00:20:20 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2736.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:20 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 109 times in the past 107.654804 secs (last seen 1.000284 secs ago PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2747.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2748.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2749.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2750.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2751.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2752.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2747.0> 2011-07-14 00:20:20 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2747.0> 2011-07-14 00:20:20 =============================================================================== ** Generic server <5901.2747.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.23646>}, init, {1310,617220,401437}, "default",#Port<5901.2731>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2747.0> 2011-07-14 00:20:20 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2747.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2628.0>] messages [] links [<5901.49.0>,<5901.2629.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2629.0> 2011-07-14 00:20:20 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2747.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2749.0> 2011-07-14 00:20:20 =============================================================================== ns_1@10.88.165.188:<5901.2749.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:20 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 111 times in the past 107.997772 secs (last seen 1.000214 secs ago PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2756.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:20 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:20 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2757.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2758.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2759.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2760.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2761.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2756.0> 2011-07-14 00:20:21 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2756.0> 2011-07-14 00:20:21 =============================================================================== ** Generic server <5901.2756.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.23722>}, init, {1310,617220,900815}, "default",#Port<5901.2732>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2756.0> 2011-07-14 00:20:21 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2756.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2628.0>] messages [] links [<5901.49.0>,<5901.2629.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2629.0> 2011-07-14 00:20:21 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2756.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:21 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 110 times in the past 108.654847 secs (last seen 1.000043 secs ago PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2765.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2766.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2767.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2768.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2769.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2770.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2765.0> 2011-07-14 00:20:21 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2765.0> 2011-07-14 00:20:21 =============================================================================== ** Generic server <5901.2765.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.23810>}, init, {1310,617221,401387}, "default",#Port<5901.2733>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2765.0> 2011-07-14 00:20:21 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2765.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2628.0>] messages [] links [<5901.49.0>,<5901.2629.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2629.0> 2011-07-14 00:20:21 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2765.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2767.0> 2011-07-14 00:20:21 =============================================================================== ns_1@10.88.165.188:<5901.2767.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:21 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 112 times in the past 108.997687 secs (last seen 0.999915 secs ago PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2777.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2778.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2779.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:21 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:21 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2780.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2781.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2782.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2777.0> 2011-07-14 00:20:22 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2777.0> 2011-07-14 00:20:22 =============================================================================== ** Generic server <5901.2777.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.23891>}, init, {1310,617221,900785}, "default",#Port<5901.2734>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2777.0> 2011-07-14 00:20:22 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2777.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2628.0>] messages [] links [<5901.49.0>,<5901.2629.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2629.0> 2011-07-14 00:20:22 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2777.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:22 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 111 times in the past 109.654936 secs (last seen 1.000089 secs ago PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2786.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2787.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2788.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2789.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2790.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2791.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2786.0> 2011-07-14 00:20:22 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2786.0> 2011-07-14 00:20:22 =============================================================================== ** Generic server <5901.2786.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.23978>}, init, {1310,617222,401521}, "default",#Port<5901.2735>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2786.0> 2011-07-14 00:20:22 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2786.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2628.0>] messages [] links [<5901.49.0>,<5901.2629.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2629.0> 2011-07-14 00:20:22 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2786.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2788.0> 2011-07-14 00:20:22 =============================================================================== ns_1@10.88.165.188:<5901.2788.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:22 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 113 times in the past 109.998203 secs (last seen 1.000516 secs ago PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2795.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2796.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2797.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2798.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2799.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2800.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2795.0> 2011-07-14 00:20:23 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2795.0> 2011-07-14 00:20:23 =============================================================================== ** Generic server <5901.2795.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.24060>}, init, {1310,617222,901311}, "default",#Port<5901.2736>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2795.0> 2011-07-14 00:20:23 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2795.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2628.0>] messages [] links [<5901.49.0>,<5901.2629.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2629.0> 2011-07-14 00:20:23 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2795.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:23 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 112 times in the past 110.657699 secs (last seen 1.002763 secs ago PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2806.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:23 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:23 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2807.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2808.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2809.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2810.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2811.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2806.0> 2011-07-14 00:20:23 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2806.0> 2011-07-14 00:20:23 =============================================================================== ** Generic server <5901.2806.0> terminating ** Last message in was list_vbuckets_prevstate ** When Server state == {state,{interval,#Ref<5901.0.0.24198>}, init, {1310,617223,404251}, "default",#Port<5901.2738>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2806.0> 2011-07-14 00:20:23 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2806.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2628.0>] messages [] links [<5901.49.0>,<5901.2629.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6671 SUPERVISOR REPORT <5901.2629.0> 2011-07-14 00:20:23 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2806.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker CRASH REPORT <5901.2818.0> 2011-07-14 00:20:23 =============================================================================== Crashing process initial_call {ns_janitor,cleanup,['Argument__1','Argument__2']} pid <5901.2818.0> registered_name [] error_info {exit,{{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, {gen_server,call, [{'ns_memcached-default','ns_1@10.88.165.188'}, list_vbuckets_prevstate,30000]}}, [{gen_server,call,3}, {ns_janitor,'-read_existing_map/4-lc$^0/1-0-',2}, {ns_janitor,read_existing_map,4}, {ns_janitor,cleanup,2}, {proc_lib,init_p_do_apply,3}]} ancestors [<5901.304.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<5901.52.0>] messages [] links [<5901.304.0>] dictionary [] trap_exit false status running heap_size 10946 stack_size 24 reductions 224 INFO REPORT <5901.227.0> 2011-07-14 00:20:23 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 114 times in the past 110.995847 secs (last seen 0.997644 secs ago INFO REPORT <5901.304.0> 2011-07-14 00:20:23 =============================================================================== ns_1@10.88.165.188:<5901.304.0>:ns_orchestrator:178: Janitor run exited for bucket "default" with reason {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, [{'ns_memcached-default', 'ns_1@10.88.165.188'}, list_vbuckets_prevstate, 30000]}} ERROR REPORT <5901.2808.0> 2011-07-14 00:20:23 =============================================================================== ns_1@10.88.165.188:<5901.2808.0>:stats_collector:94: Exception in stats collector: {exit, {noproc, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2824.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2825.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2827.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2828.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2830.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2831.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2824.0> 2011-07-14 00:20:24 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2824.0> 2011-07-14 00:20:24 =============================================================================== ** Generic server <5901.2824.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.24314>}, init, {1310,617223,900450}, "default",#Port<5901.2740>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2824.0> 2011-07-14 00:20:24 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2824.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2628.0>] messages [] links [<5901.49.0>,<5901.2629.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.2629.0> 2011-07-14 00:20:24 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2824.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:24 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 113 times in the past 111.653982 secs (last seen 0.996283 secs ago PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2835.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2836.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2837.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2838.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2839.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2840.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:24 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:24 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.2835.0> 2011-07-14 00:20:24 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2835.0> 2011-07-14 00:20:24 =============================================================================== ** Generic server <5901.2835.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.24411>}, init, {1310,617224,400508}, "default",#Port<5901.2741>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2835.0> 2011-07-14 00:20:24 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2835.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2628.0>] messages [] links [<5901.49.0>,<5901.2629.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2629.0> 2011-07-14 00:20:24 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2835.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2837.0> 2011-07-14 00:20:24 =============================================================================== ns_1@10.88.165.188:<5901.2837.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:24 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 115 times in the past 111.99828 secs (last seen 1.002433 secs ago PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2844.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2845.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2846.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2847.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2848.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2629.0> 2011-07-14 00:20:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2849.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2844.0> 2011-07-14 00:20:25 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2844.0> 2011-07-14 00:20:25 =============================================================================== ** Generic server <5901.2844.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.24487>}, init, {1310,617224,901377}, "default",#Port<5901.2742>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2844.0> 2011-07-14 00:20:25 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2844.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2628.0>] messages [] links [<5901.49.0>,<5901.2629.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2629.0> 2011-07-14 00:20:25 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2844.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker SUPERVISOR REPORT <5901.2629.0> 2011-07-14 00:20:25 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext shutdown reason reached_max_restart_intensity pid <5901.2844.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:25 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 114 times in the past 112.657859 secs (last seen 1.003877 secs ago INFO REPORT <5901.2628.0> 2011-07-14 00:20:25 =============================================================================== ns_1@10.88.165.188:<5901.2628.0>:single_bucket_sup:24: per-bucket supervisor for "default" died with reason shutdown SUPERVISOR REPORT <5901.273.0> 2011-07-14 00:20:25 =============================================================================== Reporting supervisor {local,ns_bucket_sup} Child process errorContext child_terminated reason shutdown pid <5901.2628.0> name {per_bucket_sup,"default"} mfargs {single_bucket_sup,start_link,["default"]} restart_type permanent shutdown infinity child_type supervisor PROGRESS REPORT <5901.273.0> 2011-07-14 00:20:25 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<5901.2853.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2855.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2856.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2857.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2858.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2859.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2860.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2855.0> 2011-07-14 00:20:25 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2855.0> 2011-07-14 00:20:25 =============================================================================== ** Generic server <5901.2855.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.24575>}, init, {1310,617225,404665}, "default",#Port<5901.2743>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2855.0> 2011-07-14 00:20:25 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2855.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2853.0>] messages [] links [<5901.49.0>,<5901.2854.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2854.0> 2011-07-14 00:20:25 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2855.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2857.0> 2011-07-14 00:20:25 =============================================================================== ns_1@10.88.165.188:<5901.2857.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:25 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 116 times in the past 112.998007 secs (last seen 0.999727 secs ago PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2867.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2868.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2869.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:25 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:25 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2870.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2871.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2872.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2867.0> 2011-07-14 00:20:26 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2867.0> 2011-07-14 00:20:26 =============================================================================== ** Generic server <5901.2867.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.24656>}, init, {1310,617225,901112}, "default",#Port<5901.2744>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2867.0> 2011-07-14 00:20:26 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2867.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2853.0>] messages [] links [<5901.49.0>,<5901.2854.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2854.0> 2011-07-14 00:20:26 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2867.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:26 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 115 times in the past 113.657871 secs (last seen 1.000012 secs ago PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2878.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2879.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2880.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2881.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2882.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2883.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2878.0> 2011-07-14 00:20:26 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2878.0> 2011-07-14 00:20:26 =============================================================================== ** Generic server <5901.2878.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.24806>}, init, {1310,617226,404426}, "default",#Port<5901.2746>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2878.0> 2011-07-14 00:20:26 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2878.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2853.0>] messages [] links [<5901.49.0>,<5901.2854.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2854.0> 2011-07-14 00:20:26 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2878.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2880.0> 2011-07-14 00:20:26 =============================================================================== ns_1@10.88.165.188:<5901.2880.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:26 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 117 times in the past 113.998082 secs (last seen 1.000075 secs ago PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2887.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2888.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2889.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2890.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2891.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2892.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2887.0> 2011-07-14 00:20:27 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2887.0> 2011-07-14 00:20:27 =============================================================================== ** Generic server <5901.2887.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.24889>}, init, {1310,617226,901089}, "default",#Port<5901.2747>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2887.0> 2011-07-14 00:20:27 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2887.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2853.0>] messages [] links [<5901.49.0>,<5901.2854.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2854.0> 2011-07-14 00:20:27 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2887.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:27 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 116 times in the past 114.657883 secs (last seen 1.000012 secs ago PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2896.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2897.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2898.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:27 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:27 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2899.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2900.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2901.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2896.0> 2011-07-14 00:20:27 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2896.0> 2011-07-14 00:20:27 =============================================================================== ** Generic server <5901.2896.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.24971>}, init, {1310,617227,404423}, "default",#Port<5901.2748>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2896.0> 2011-07-14 00:20:27 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2896.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2853.0>] messages [] links [<5901.49.0>,<5901.2854.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2854.0> 2011-07-14 00:20:27 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2896.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2898.0> 2011-07-14 00:20:27 =============================================================================== ns_1@10.88.165.188:<5901.2898.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:27 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 118 times in the past 114.998154 secs (last seen 1.000072 secs ago PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2908.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2909.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2910.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2911.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2912.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2913.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2908.0> 2011-07-14 00:20:28 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2908.0> 2011-07-14 00:20:28 =============================================================================== ** Generic server <5901.2908.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.25056>}, init, {1310,617227,901252}, "default",#Port<5901.2749>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2908.0> 2011-07-14 00:20:28 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2908.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2853.0>] messages [] links [<5901.49.0>,<5901.2854.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2854.0> 2011-07-14 00:20:28 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2908.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:28 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 117 times in the past 115.655251 secs (last seen 0.997368 secs ago PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2917.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2918.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2919.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2920.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2921.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2922.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2917.0> 2011-07-14 00:20:28 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2917.0> 2011-07-14 00:20:28 =============================================================================== ** Generic server <5901.2917.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.25144>}, init, {1310,617228,401700}, "default",#Port<5901.2750>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2917.0> 2011-07-14 00:20:28 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2917.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2853.0>] messages [] links [<5901.49.0>,<5901.2854.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6665 SUPERVISOR REPORT <5901.2854.0> 2011-07-14 00:20:28 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2917.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2919.0> 2011-07-14 00:20:28 =============================================================================== ns_1@10.88.165.188:<5901.2919.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:28 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 119 times in the past 115.998119 secs (last seen 0.999965 secs ago PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2930.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:28 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:28 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2931.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2933.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2934.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2936.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2937.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2930.0> 2011-07-14 00:20:29 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2930.0> 2011-07-14 00:20:29 =============================================================================== ** Generic server <5901.2930.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.25313>}, init, {1310,617228,901173}, "default",#Port<5901.2752>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2930.0> 2011-07-14 00:20:29 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2930.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2853.0>] messages [] links [<5901.49.0>,<5901.2854.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.2854.0> 2011-07-14 00:20:29 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2930.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:29 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 118 times in the past 116.655265 secs (last seen 1.000014 secs ago PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2942.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2943.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2944.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2945.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2946.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2947.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2942.0> 2011-07-14 00:20:29 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2942.0> 2011-07-14 00:20:29 =============================================================================== ** Generic server <5901.2942.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.25447>}, init, {1310,617229,401867}, "default",#Port<5901.2754>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2942.0> 2011-07-14 00:20:29 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2942.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2853.0>] messages [] links [<5901.49.0>,<5901.2854.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2854.0> 2011-07-14 00:20:29 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2942.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2944.0> 2011-07-14 00:20:29 =============================================================================== ns_1@10.88.165.188:<5901.2944.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:29 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 120 times in the past 116.99823 secs (last seen 1.000111 secs ago PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2955.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2956.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2957.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:29 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:29 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2958.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2959.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2960.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2955.0> 2011-07-14 00:20:30 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2955.0> 2011-07-14 00:20:30 =============================================================================== ** Generic server <5901.2955.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.25548>}, init, {1310,617229,901385}, "default",#Port<5901.2755>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2955.0> 2011-07-14 00:20:30 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2955.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2853.0>] messages [] links [<5901.49.0>,<5901.2854.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2854.0> 2011-07-14 00:20:30 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2955.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:30 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 119 times in the past 117.655342 secs (last seen 1.000077 secs ago PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2964.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2965.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2966.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2967.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2968.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2969.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2964.0> 2011-07-14 00:20:30 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2964.0> 2011-07-14 00:20:30 =============================================================================== ** Generic server <5901.2964.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.25634>}, init, {1310,617230,401847}, "default",#Port<5901.2756>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2964.0> 2011-07-14 00:20:30 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2964.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2853.0>] messages [] links [<5901.49.0>,<5901.2854.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2854.0> 2011-07-14 00:20:30 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2964.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2966.0> 2011-07-14 00:20:30 =============================================================================== ns_1@10.88.165.188:<5901.2966.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:30 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 121 times in the past 117.998508 secs (last seen 1.000278 secs ago PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2973.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2974.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2975.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2976.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2977.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2978.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2973.0> 2011-07-14 00:20:31 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2973.0> 2011-07-14 00:20:31 =============================================================================== ** Generic server <5901.2973.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.25716>}, init, {1310,617230,902762}, "default",#Port<5901.2757>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2973.0> 2011-07-14 00:20:31 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2973.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2853.0>] messages [] links [<5901.49.0>,<5901.2854.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2854.0> 2011-07-14 00:20:31 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2973.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:31 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 120 times in the past 118.658242 secs (last seen 1.0029 secs ago PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2982.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:31 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:31 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2983.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2984.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2985.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2986.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2987.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2982.0> 2011-07-14 00:20:31 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2982.0> 2011-07-14 00:20:31 =============================================================================== ** Generic server <5901.2982.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.25799>}, init, {1310,617231,404831}, "default",#Port<5901.2758>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2982.0> 2011-07-14 00:20:31 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2982.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2853.0>] messages [] links [<5901.49.0>,<5901.2854.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2854.0> 2011-07-14 00:20:31 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2982.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.2984.0> 2011-07-14 00:20:31 =============================================================================== ns_1@10.88.165.188:<5901.2984.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:31 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 122 times in the past 118.998332 secs (last seen 0.999824 secs ago PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2994.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2995.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2996.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2997.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2998.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.2999.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.2994.0> 2011-07-14 00:20:32 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.2994.0> 2011-07-14 00:20:32 =============================================================================== ** Generic server <5901.2994.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.25886>}, init, {1310,617231,901406}, "default",#Port<5901.2759>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.2994.0> 2011-07-14 00:20:32 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.2994.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2853.0>] messages [] links [<5901.49.0>,<5901.2854.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2854.0> 2011-07-14 00:20:32 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.2994.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:32 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 121 times in the past 119.655511 secs (last seen 0.997269 secs ago PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3004.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3005.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3006.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3007.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3008.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3009.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:32 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:32 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.3004.0> 2011-07-14 00:20:32 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3004.0> 2011-07-14 00:20:32 =============================================================================== ** Generic server <5901.3004.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.26012>}, init, {1310,617232,402075}, "default",#Port<5901.2761>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3004.0> 2011-07-14 00:20:32 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3004.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2853.0>] messages [] links [<5901.49.0>,<5901.2854.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2854.0> 2011-07-14 00:20:32 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3004.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3006.0> 2011-07-14 00:20:32 =============================================================================== ns_1@10.88.165.188:<5901.3006.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:32 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 123 times in the past 119.998472 secs (last seen 1.00014 secs ago PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3014.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3015.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3016.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3017.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3018.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3019.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3014.0> 2011-07-14 00:20:33 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3014.0> 2011-07-14 00:20:33 =============================================================================== ** Generic server <5901.3014.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.26108>}, init, {1310,617232,902474}, "default",#Port<5901.2762>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3014.0> 2011-07-14 00:20:33 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3014.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2853.0>] messages [] links [<5901.49.0>,<5901.2854.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2854.0> 2011-07-14 00:20:33 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3014.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:33 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 122 times in the past 120.658377 secs (last seen 1.002866 secs ago PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3023.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3024.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3025.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3026.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3027.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3028.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3023.0> 2011-07-14 00:20:33 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3023.0> 2011-07-14 00:20:33 =============================================================================== ** Generic server <5901.3023.0> terminating ** Last message in was list_vbuckets_prevstate ** When Server state == {state,{interval,#Ref<5901.0.0.26197>}, init, {1310,617233,404929}, "default",#Port<5901.2763>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3023.0> 2011-07-14 00:20:33 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3023.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2853.0>] messages [{'$gen_call',{<5901.3025.0>,#Ref<5901.0.0.26298>},{stats,<<>>}}] links [<5901.49.0>,<5901.2854.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6671 SUPERVISOR REPORT <5901.2854.0> 2011-07-14 00:20:33 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3023.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker CRASH REPORT <5901.3035.0> 2011-07-14 00:20:33 =============================================================================== Crashing process initial_call {ns_janitor,cleanup,['Argument__1','Argument__2']} pid <5901.3035.0> registered_name [] error_info {exit,{{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, {gen_server,call, [{'ns_memcached-default','ns_1@10.88.165.188'}, list_vbuckets_prevstate,30000]}}, [{gen_server,call,3}, {ns_janitor,'-read_existing_map/4-lc$^0/1-0-',2}, {ns_janitor,read_existing_map,4}, {ns_janitor,cleanup,2}, {proc_lib,init_p_do_apply,3}]} ancestors [<5901.304.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<5901.52.0>] messages [] links [<5901.304.0>] dictionary [] trap_exit false status running heap_size 10946 stack_size 24 reductions 224 ERROR REPORT <5901.3025.0> 2011-07-14 00:20:33 =============================================================================== ns_1@10.88.165.188:<5901.3025.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:33 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 124 times in the past 120.997197 secs (last seen 0.998725 secs ago INFO REPORT <5901.304.0> 2011-07-14 00:20:33 =============================================================================== ns_1@10.88.165.188:<5901.304.0>:ns_orchestrator:178: Janitor run exited for bucket "default" with reason {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, [{'ns_memcached-default', 'ns_1@10.88.165.188'}, list_vbuckets_prevstate, 30000]}} PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3041.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3042.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3044.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3045.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3046.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3047.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:33 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:33 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.3041.0> 2011-07-14 00:20:34 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3041.0> 2011-07-14 00:20:34 =============================================================================== ** Generic server <5901.3041.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.26308>}, init, {1310,617233,900281}, "default",#Port<5901.2765>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3041.0> 2011-07-14 00:20:34 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3041.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2853.0>] messages [] links [<5901.49.0>,<5901.2854.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.2854.0> 2011-07-14 00:20:34 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3041.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:34 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 123 times in the past 121.655147 secs (last seen 0.99677 secs ago PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3052.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3053.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3054.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3055.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3056.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3057.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3052.0> 2011-07-14 00:20:34 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3052.0> 2011-07-14 00:20:34 =============================================================================== ** Generic server <5901.3052.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.26404>}, init, {1310,617234,401814}, "default",#Port<5901.2766>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3052.0> 2011-07-14 00:20:34 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3052.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2853.0>] messages [] links [<5901.49.0>,<5901.2854.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2854.0> 2011-07-14 00:20:34 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3052.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3054.0> 2011-07-14 00:20:34 =============================================================================== ns_1@10.88.165.188:<5901.3054.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:34 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 125 times in the past 121.998636 secs (last seen 1.001439 secs ago PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3061.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3062.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3063.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3064.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3065.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3066.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3061.0> 2011-07-14 00:20:35 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3061.0> 2011-07-14 00:20:35 =============================================================================== ** Generic server <5901.3061.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.26486>}, init, {1310,617234,901612}, "default",#Port<5901.2767>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3061.0> 2011-07-14 00:20:35 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3061.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2853.0>] messages [] links [<5901.49.0>,<5901.2854.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.2854.0> 2011-07-14 00:20:35 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3061.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:35 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 124 times in the past 122.65578 secs (last seen 1.000633 secs ago PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3071.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3072.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3073.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:35 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:35 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3074.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3075.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.2854.0> 2011-07-14 00:20:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3076.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3071.0> 2011-07-14 00:20:35 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3071.0> 2011-07-14 00:20:35 =============================================================================== ** Generic server <5901.3071.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.26604>}, init, {1310,617235,402325}, "default",#Port<5901.2769>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3071.0> 2011-07-14 00:20:35 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3071.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.2853.0>] messages [] links [<5901.49.0>,<5901.2854.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.2854.0> 2011-07-14 00:20:35 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3071.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker SUPERVISOR REPORT <5901.2854.0> 2011-07-14 00:20:35 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext shutdown reason reached_max_restart_intensity pid <5901.3071.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3073.0> 2011-07-14 00:20:35 =============================================================================== ns_1@10.88.165.188:<5901.3073.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:35 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 126 times in the past 122.99856 secs (last seen 0.999924 secs ago INFO REPORT <5901.2853.0> 2011-07-14 00:20:35 =============================================================================== ns_1@10.88.165.188:<5901.2853.0>:single_bucket_sup:24: per-bucket supervisor for "default" died with reason shutdown SUPERVISOR REPORT <5901.273.0> 2011-07-14 00:20:35 =============================================================================== Reporting supervisor {local,ns_bucket_sup} Child process errorContext child_terminated reason shutdown pid <5901.2853.0> name {per_bucket_sup,"default"} mfargs {single_bucket_sup,start_link,["default"]} restart_type permanent shutdown infinity child_type supervisor PROGRESS REPORT <5901.273.0> 2011-07-14 00:20:35 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<5901.3084.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3086.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3087.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3088.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3089.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3090.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3091.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3086.0> 2011-07-14 00:20:36 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3086.0> 2011-07-14 00:20:36 =============================================================================== ** Generic server <5901.3086.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.26710>}, init, {1310,617235,901828}, "default",#Port<5901.2770>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3086.0> 2011-07-14 00:20:36 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3086.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3084.0>] messages [] links [<5901.49.0>,<5901.3085.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3085.0> 2011-07-14 00:20:36 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3086.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:36 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 125 times in the past 123.655791 secs (last seen 1.000011 secs ago PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3095.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3096.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3097.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3098.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3099.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3100.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3095.0> 2011-07-14 00:20:36 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3095.0> 2011-07-14 00:20:36 =============================================================================== ** Generic server <5901.3095.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.26798>}, init, {1310,617236,402380}, "default",#Port<5901.2771>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3095.0> 2011-07-14 00:20:36 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3095.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3084.0>] messages [] links [<5901.49.0>,<5901.3085.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3085.0> 2011-07-14 00:20:36 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3095.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3097.0> 2011-07-14 00:20:36 =============================================================================== ns_1@10.88.165.188:<5901.3097.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:36 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 127 times in the past 123.99881 secs (last seen 1.00025 secs ago PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3104.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3105.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3106.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:36 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:36 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3107.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3108.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3109.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3104.0> 2011-07-14 00:20:37 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3104.0> 2011-07-14 00:20:37 =============================================================================== ** Generic server <5901.3104.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.26874>}, init, {1310,617236,901938}, "default",#Port<5901.2772>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3104.0> 2011-07-14 00:20:37 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3104.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3084.0>] messages [] links [<5901.49.0>,<5901.3085.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3085.0> 2011-07-14 00:20:37 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3104.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:37 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 126 times in the past 124.658543 secs (last seen 1.002752 secs ago PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3113.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3114.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3115.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3116.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3117.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3118.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3113.0> 2011-07-14 00:20:37 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3113.0> 2011-07-14 00:20:37 =============================================================================== ** Generic server <5901.3113.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.26960>}, init, {1310,617237,405049}, "default",#Port<5901.2773>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3113.0> 2011-07-14 00:20:37 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3113.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3084.0>] messages [] links [<5901.49.0>,<5901.3085.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3085.0> 2011-07-14 00:20:37 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3113.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3115.0> 2011-07-14 00:20:37 =============================================================================== ns_1@10.88.165.188:<5901.3115.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:37 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 128 times in the past 124.998757 secs (last seen 0.999947 secs ago PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3125.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3126.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3127.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3128.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3129.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3130.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3125.0> 2011-07-14 00:20:38 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3125.0> 2011-07-14 00:20:38 =============================================================================== ** Generic server <5901.3125.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.27048>}, init, {1310,617237,901932}, "default",#Port<5901.2774>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3125.0> 2011-07-14 00:20:38 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3125.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3084.0>] messages [] links [<5901.49.0>,<5901.3085.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3085.0> 2011-07-14 00:20:38 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3125.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:38 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 127 times in the past 125.655859 secs (last seen 0.997316 secs ago PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3135.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3136.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3137.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3138.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3139.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3140.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:38 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:38 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.239.0> 2011-07-14 00:20:38 =============================================================================== ns_1@10.88.165.188:<5901.239.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.3135.0> 2011-07-14 00:20:38 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3135.0> 2011-07-14 00:20:38 =============================================================================== ** Generic server <5901.3135.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.27179>}, init, {1310,617238,402418}, "default",#Port<5901.2776>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3135.0> 2011-07-14 00:20:38 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3135.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3084.0>] messages [] links [<5901.49.0>,<5901.3085.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6665 SUPERVISOR REPORT <5901.3085.0> 2011-07-14 00:20:38 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3135.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3137.0> 2011-07-14 00:20:38 =============================================================================== ns_1@10.88.165.188:<5901.3137.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:38 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 129 times in the past 125.997315 secs (last seen 0.998558 secs ago PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3149.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3151.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3152.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3153.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3154.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3155.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3149.0> 2011-07-14 00:20:39 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3149.0> 2011-07-14 00:20:39 =============================================================================== ** Generic server <5901.3149.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.27287>}, init, {1310,617238,900420}, "default",#Port<5901.2778>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3149.0> 2011-07-14 00:20:39 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3149.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3084.0>] messages [] links [<5901.49.0>,<5901.3085.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.3085.0> 2011-07-14 00:20:39 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3149.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:39 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 128 times in the past 126.655128 secs (last seen 0.999269 secs ago PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3161.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3162.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3163.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3164.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3165.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3166.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3161.0> 2011-07-14 00:20:39 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3161.0> 2011-07-14 00:20:39 =============================================================================== ** Generic server <5901.3161.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.27397>}, init, {1310,617239,401688}, "default",#Port<5901.2779>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3161.0> 2011-07-14 00:20:39 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3161.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3084.0>] messages [] links [<5901.49.0>,<5901.3085.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3085.0> 2011-07-14 00:20:39 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3161.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3163.0> 2011-07-14 00:20:39 =============================================================================== ns_1@10.88.165.188:<5901.3163.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:39 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 130 times in the past 126.998865 secs (last seen 1.00155 secs ago PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3173.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:39 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:39 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3174.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3175.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3176.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3177.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3178.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3173.0> 2011-07-14 00:20:40 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3173.0> 2011-07-14 00:20:40 =============================================================================== ** Generic server <5901.3173.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.27482>}, init, {1310,617239,903008}, "default",#Port<5901.2780>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3173.0> 2011-07-14 00:20:40 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3173.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3084.0>] messages [] links [<5901.49.0>,<5901.3085.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3085.0> 2011-07-14 00:20:40 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3173.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:40 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 129 times in the past 127.656157 secs (last seen 1.001029 secs ago PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3182.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3183.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3184.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3185.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3186.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3187.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3182.0> 2011-07-14 00:20:40 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3182.0> 2011-07-14 00:20:40 =============================================================================== ** Generic server <5901.3182.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.27571>}, init, {1310,617240,402958}, "default",#Port<5901.2781>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3182.0> 2011-07-14 00:20:40 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3182.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3084.0>] messages [] links [<5901.49.0>,<5901.3085.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3085.0> 2011-07-14 00:20:40 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3182.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3184.0> 2011-07-14 00:20:40 =============================================================================== ns_1@10.88.165.188:<5901.3184.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:40 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 131 times in the past 127.999047 secs (last seen 1.000182 secs ago PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3191.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3192.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3193.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:40 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:40 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3194.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3195.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3196.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3191.0> 2011-07-14 00:20:41 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3191.0> 2011-07-14 00:20:41 =============================================================================== ** Generic server <5901.3191.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.27647>}, init, {1310,617240,902180}, "default",#Port<5901.2782>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3191.0> 2011-07-14 00:20:41 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3191.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3084.0>] messages [] links [<5901.49.0>,<5901.3085.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3085.0> 2011-07-14 00:20:41 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3191.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:41 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 130 times in the past 128.658913 secs (last seen 1.002756 secs ago PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3201.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3202.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3203.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3204.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3205.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3206.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3201.0> 2011-07-14 00:20:41 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3201.0> 2011-07-14 00:20:41 =============================================================================== ** Generic server <5901.3201.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.27769>}, init, {1310,617241,405527}, "default",#Port<5901.2784>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3201.0> 2011-07-14 00:20:41 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3201.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3084.0>] messages [] links [<5901.49.0>,<5901.3085.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3085.0> 2011-07-14 00:20:41 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3201.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3203.0> 2011-07-14 00:20:41 =============================================================================== ns_1@10.88.165.188:<5901.3203.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:41 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 132 times in the past 128.998967 secs (last seen 0.99992 secs ago PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3213.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3214.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3215.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3216.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3217.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3218.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3213.0> 2011-07-14 00:20:42 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3213.0> 2011-07-14 00:20:42 =============================================================================== ** Generic server <5901.3213.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.27852>}, init, {1310,617241,902952}, "default",#Port<5901.2785>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3213.0> 2011-07-14 00:20:42 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3213.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3084.0>] messages [] links [<5901.49.0>,<5901.3085.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3085.0> 2011-07-14 00:20:42 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3213.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:42 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 131 times in the past 129.658809 secs (last seen 0.999896 secs ago PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3223.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:42 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:42 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3224.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3225.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3226.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3227.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3228.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3223.0> 2011-07-14 00:20:42 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3223.0> 2011-07-14 00:20:42 =============================================================================== ** Generic server <5901.3223.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.27959>}, init, {1310,617242,405312}, "default",#Port<5901.2786>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3223.0> 2011-07-14 00:20:42 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3223.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3084.0>] messages [] links [<5901.49.0>,<5901.3085.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3085.0> 2011-07-14 00:20:42 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3223.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3225.0> 2011-07-14 00:20:42 =============================================================================== ns_1@10.88.165.188:<5901.3225.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:42 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 133 times in the past 129.999106 secs (last seen 1.000139 secs ago PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3232.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3233.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3234.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3235.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3236.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3237.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3232.0> 2011-07-14 00:20:43 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3232.0> 2011-07-14 00:20:43 =============================================================================== ** Generic server <5901.3232.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.28039>}, init, {1310,617242,902116}, "default",#Port<5901.2787>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3232.0> 2011-07-14 00:20:43 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3232.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3084.0>] messages [] links [<5901.49.0>,<5901.3085.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3085.0> 2011-07-14 00:20:43 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3232.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:43 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 132 times in the past 130.658878 secs (last seen 1.000069 secs ago PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3241.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3242.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3243.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3244.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3245.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3246.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3241.0> 2011-07-14 00:20:43 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3241.0> 2011-07-14 00:20:43 =============================================================================== ** Generic server <5901.3241.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.28123>}, init, {1310,617243,405364}, "default",#Port<5901.2788>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3241.0> 2011-07-14 00:20:43 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3241.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3084.0>] messages [] links [<5901.49.0>,<5901.3085.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6671 SUPERVISOR REPORT <5901.3085.0> 2011-07-14 00:20:43 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3241.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3243.0> 2011-07-14 00:20:43 =============================================================================== ns_1@10.88.165.188:<5901.3243.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:43 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 134 times in the past 130.997488 secs (last seen 0.998382 secs ago PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3259.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:43 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:43 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3261.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3263.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3264.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3266.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3267.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3259.0> 2011-07-14 00:20:43 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3259.0> 2011-07-14 00:20:43 =============================================================================== ** Generic server <5901.3259.0> terminating ** Last message in was list_vbuckets_prevstate ** When Server state == {state,{interval,#Ref<5901.0.0.28238>}, init, {1310,617243,901399}, "default",#Port<5901.2791>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3259.0> 2011-07-14 00:20:43 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3259.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3084.0>] messages [] links [<5901.49.0>,<5901.3085.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6701 SUPERVISOR REPORT <5901.3085.0> 2011-07-14 00:20:43 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3259.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker CRASH REPORT <5901.3253.0> 2011-07-14 00:20:43 =============================================================================== Crashing process initial_call {ns_janitor,cleanup,['Argument__1','Argument__2']} pid <5901.3253.0> registered_name [] error_info {exit,{{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, {gen_server,call, [{'ns_memcached-default','ns_1@10.88.165.188'}, list_vbuckets_prevstate,30000]}}, [{gen_server,call,3}, {ns_janitor,'-read_existing_map/4-lc$^0/1-0-',2}, {ns_janitor,read_existing_map,4}, {ns_janitor,cleanup,2}, {proc_lib,init_p_do_apply,3}]} ancestors [<5901.304.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<5901.52.0>] messages [] links [<5901.304.0>] dictionary [] trap_exit false status running heap_size 10946 stack_size 24 reductions 224 INFO REPORT <5901.227.0> 2011-07-14 00:20:43 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 135 times in the past 131.009207 secs (last seen 0.011719 secs ago INFO REPORT <5901.304.0> 2011-07-14 00:20:43 =============================================================================== ns_1@10.88.165.188:<5901.304.0>:ns_orchestrator:178: Janitor run exited for bucket "default" with reason {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, [{'ns_memcached-default', 'ns_1@10.88.165.188'}, list_vbuckets_prevstate, 30000]}} PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3268.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3269.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] SUPERVISOR REPORT <5901.258.0> 2011-07-14 00:20:43 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.88.165.188'}, {latest,minute,1}]}} pid <5901.1953.0> name menelaus_web_alerts_srv mfargs {menelaus_web_alerts_srv,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3271.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.258.0> 2011-07-14 00:20:43 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<5901.3272.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3273.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3274.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3275.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3268.0> 2011-07-14 00:20:44 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3268.0> 2011-07-14 00:20:44 =============================================================================== ** Generic server <5901.3268.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.28389>}, init, {1310,617243,911352}, "default",#Port<5901.2792>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3268.0> 2011-07-14 00:20:44 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3268.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3084.0>] messages [] links [<5901.49.0>,<5901.3085.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3085.0> 2011-07-14 00:20:44 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3268.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:44 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 133 times in the past 131.667016 secs (last seen 1.008138 secs ago PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3279.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3280.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3281.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3282.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3283.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3284.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3279.0> 2011-07-14 00:20:44 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3279.0> 2011-07-14 00:20:44 =============================================================================== ** Generic server <5901.3279.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.28479>}, init, {1310,617244,413548}, "default",#Port<5901.2793>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3279.0> 2011-07-14 00:20:44 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3279.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3084.0>] messages [] links [<5901.49.0>,<5901.3085.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3085.0> 2011-07-14 00:20:44 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3279.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3281.0> 2011-07-14 00:20:44 =============================================================================== ns_1@10.88.165.188:<5901.3281.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:44 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 136 times in the past 131.999239 secs (last seen 0.990032 secs ago PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3288.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3289.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3290.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3291.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3292.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3293.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:44 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:44 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.3288.0> 2011-07-14 00:20:45 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3288.0> 2011-07-14 00:20:45 =============================================================================== ** Generic server <5901.3288.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.28555>}, init, {1310,617244,901954}, "default",#Port<5901.2794>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3288.0> 2011-07-14 00:20:45 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3288.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3084.0>] messages [] links [<5901.49.0>,<5901.3085.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3085.0> 2011-07-14 00:20:45 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3288.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:45 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 134 times in the past 132.656279 secs (last seen 0.989263 secs ago PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3298.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3299.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3300.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3301.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3302.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3085.0> 2011-07-14 00:20:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3303.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3298.0> 2011-07-14 00:20:45 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3298.0> 2011-07-14 00:20:45 =============================================================================== ** Generic server <5901.3298.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.28667>}, init, {1310,617245,402790}, "default",#Port<5901.2795>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3298.0> 2011-07-14 00:20:45 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3298.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3084.0>] messages [] links [<5901.49.0>,<5901.3085.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3085.0> 2011-07-14 00:20:45 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3298.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker SUPERVISOR REPORT <5901.3085.0> 2011-07-14 00:20:45 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext shutdown reason reached_max_restart_intensity pid <5901.3298.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3300.0> 2011-07-14 00:20:45 =============================================================================== ns_1@10.88.165.188:<5901.3300.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:45 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 137 times in the past 132.999175 secs (last seen 0.999936 secs ago INFO REPORT <5901.3084.0> 2011-07-14 00:20:45 =============================================================================== ns_1@10.88.165.188:<5901.3084.0>:single_bucket_sup:24: per-bucket supervisor for "default" died with reason shutdown SUPERVISOR REPORT <5901.273.0> 2011-07-14 00:20:45 =============================================================================== Reporting supervisor {local,ns_bucket_sup} Child process errorContext child_terminated reason shutdown pid <5901.3084.0> name {per_bucket_sup,"default"} mfargs {single_bucket_sup,start_link,["default"]} restart_type permanent shutdown infinity child_type supervisor PROGRESS REPORT <5901.273.0> 2011-07-14 00:20:45 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<5901.3310.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3312.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3313.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3314.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3315.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3316.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3317.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3312.0> 2011-07-14 00:20:46 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3312.0> 2011-07-14 00:20:46 =============================================================================== ** Generic server <5901.3312.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.28750>}, init, {1310,617245,902485}, "default",#Port<5901.2796>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3312.0> 2011-07-14 00:20:46 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3312.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3310.0>] messages [] links [<5901.49.0>,<5901.3311.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3311.0> 2011-07-14 00:20:46 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3312.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:46 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 135 times in the past 133.656347 secs (last seen 1.000068 secs ago PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3321.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3322.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3323.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:46 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:46 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3324.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3325.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3326.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3321.0> 2011-07-14 00:20:46 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3321.0> 2011-07-14 00:20:46 =============================================================================== ** Generic server <5901.3321.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.28836>}, init, {1310,617246,402816}, "default",#Port<5901.2797>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3321.0> 2011-07-14 00:20:46 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3321.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3310.0>] messages [] links [<5901.49.0>,<5901.3311.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3311.0> 2011-07-14 00:20:46 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3321.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3323.0> 2011-07-14 00:20:46 =============================================================================== ns_1@10.88.165.188:<5901.3323.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:46 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 138 times in the past 133.999267 secs (last seen 1.000092 secs ago PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3330.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3331.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3332.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3333.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3334.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3335.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3330.0> 2011-07-14 00:20:47 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3330.0> 2011-07-14 00:20:47 =============================================================================== ** Generic server <5901.3330.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.28917>}, init, {1310,617246,902472}, "default",#Port<5901.2798>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3330.0> 2011-07-14 00:20:47 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3330.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3310.0>] messages [] links [<5901.49.0>,<5901.3311.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3311.0> 2011-07-14 00:20:47 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3330.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:47 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 136 times in the past 134.65637 secs (last seen 1.000023 secs ago PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3340.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3341.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3342.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3343.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3344.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3345.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3340.0> 2011-07-14 00:20:47 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3340.0> 2011-07-14 00:20:47 =============================================================================== ** Generic server <5901.3340.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.29037>}, init, {1310,617247,402886}, "default",#Port<5901.2800>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3340.0> 2011-07-14 00:20:47 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3340.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3310.0>] messages [] links [<5901.49.0>,<5901.3311.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3311.0> 2011-07-14 00:20:47 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3340.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3342.0> 2011-07-14 00:20:47 =============================================================================== ns_1@10.88.165.188:<5901.3342.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:47 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 139 times in the past 134.999426 secs (last seen 1.000159 secs ago PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3352.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3353.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3354.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:47 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:47 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3355.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3356.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3357.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3352.0> 2011-07-14 00:20:48 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3352.0> 2011-07-14 00:20:48 =============================================================================== ** Generic server <5901.3352.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.29122>}, init, {1310,617247,902470}, "default",#Port<5901.2801>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3352.0> 2011-07-14 00:20:48 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3352.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3310.0>] messages [] links [<5901.49.0>,<5901.3311.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3311.0> 2011-07-14 00:20:48 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3352.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:48 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 137 times in the past 135.659303 secs (last seen 1.002933 secs ago PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3362.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3363.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3364.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3365.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3366.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3367.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3362.0> 2011-07-14 00:20:48 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3362.0> 2011-07-14 00:20:48 =============================================================================== ** Generic server <5901.3362.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.29228>}, init, {1310,617248,405949}, "default",#Port<5901.2802>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3362.0> 2011-07-14 00:20:48 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3362.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3310.0>] messages [] links [<5901.49.0>,<5901.3311.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6665 SUPERVISOR REPORT <5901.3311.0> 2011-07-14 00:20:48 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3362.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3364.0> 2011-07-14 00:20:48 =============================================================================== ns_1@10.88.165.188:<5901.3364.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:48 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 140 times in the past 135.999444 secs (last seen 1.000018 secs ago PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3374.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3375.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3377.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3378.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3379.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3380.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3374.0> 2011-07-14 00:20:49 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3374.0> 2011-07-14 00:20:49 =============================================================================== ** Generic server <5901.3374.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.29322>}, init, {1310,617248,902526}, "default",#Port<5901.2803>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3374.0> 2011-07-14 00:20:49 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3374.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3310.0>] messages [] links [<5901.49.0>,<5901.3311.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.3311.0> 2011-07-14 00:20:49 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3374.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:49 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 138 times in the past 136.656658 secs (last seen 0.997355 secs ago PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3385.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3386.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3387.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3388.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3389.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3390.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:49 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:49 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.3385.0> 2011-07-14 00:20:49 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3385.0> 2011-07-14 00:20:49 =============================================================================== ** Generic server <5901.3385.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.29418>}, init, {1310,617249,403254}, "default",#Port<5901.2804>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3385.0> 2011-07-14 00:20:49 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3385.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3310.0>] messages [] links [<5901.49.0>,<5901.3311.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3311.0> 2011-07-14 00:20:49 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3385.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3387.0> 2011-07-14 00:20:49 =============================================================================== ns_1@10.88.165.188:<5901.3387.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:49 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 141 times in the past 136.999511 secs (last seen 1.000067 secs ago PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3397.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3398.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3399.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3400.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3401.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3402.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3397.0> 2011-07-14 00:20:50 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3397.0> 2011-07-14 00:20:50 =============================================================================== ** Generic server <5901.3397.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.29499>}, init, {1310,617249,902633}, "default",#Port<5901.2805>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3397.0> 2011-07-14 00:20:50 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3397.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3310.0>] messages [] links [<5901.49.0>,<5901.3311.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3311.0> 2011-07-14 00:20:50 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3397.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:50 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 139 times in the past 137.656634 secs (last seen 0.999976 secs ago PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3407.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3408.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3409.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3410.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3411.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3412.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3407.0> 2011-07-14 00:20:50 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3407.0> 2011-07-14 00:20:50 =============================================================================== ** Generic server <5901.3407.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.29631>}, init, {1310,617250,403188}, "default",#Port<5901.2807>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3407.0> 2011-07-14 00:20:50 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3407.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3310.0>] messages [] links [<5901.49.0>,<5901.3311.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3311.0> 2011-07-14 00:20:50 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3407.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3409.0> 2011-07-14 00:20:50 =============================================================================== ns_1@10.88.165.188:<5901.3409.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:50 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 142 times in the past 138.000474 secs (last seen 1.000963 secs ago INFO REPORT <5901.269.0> 2011-07-14 00:20:50 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:50 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3416.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3417.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3418.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3419.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3420.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3421.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3416.0> 2011-07-14 00:20:51 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3416.0> 2011-07-14 00:20:51 =============================================================================== ** Generic server <5901.3416.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.29711>}, init, {1310,617250,904108}, "default",#Port<5901.2808>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3416.0> 2011-07-14 00:20:51 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3416.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3310.0>] messages [] links [<5901.49.0>,<5901.3311.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3311.0> 2011-07-14 00:20:51 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3416.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:51 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 140 times in the past 138.659899 secs (last seen 1.003265 secs ago PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3425.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3426.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3427.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3428.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3429.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3430.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3425.0> 2011-07-14 00:20:51 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3425.0> 2011-07-14 00:20:51 =============================================================================== ** Generic server <5901.3425.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.29794>}, init, {1310,617251,406544}, "default",#Port<5901.2809>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3425.0> 2011-07-14 00:20:51 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3425.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3310.0>] messages [] links [<5901.49.0>,<5901.3311.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3311.0> 2011-07-14 00:20:51 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3425.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3427.0> 2011-07-14 00:20:51 =============================================================================== ns_1@10.88.165.188:<5901.3427.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:51 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 143 times in the past 138.999612 secs (last seen 0.999138 secs ago PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3438.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3439.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3440.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3441.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3442.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3443.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3438.0> 2011-07-14 00:20:52 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3438.0> 2011-07-14 00:20:52 =============================================================================== ** Generic server <5901.3438.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.29901>}, init, {1310,617251,902710}, "default",#Port<5901.2810>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3438.0> 2011-07-14 00:20:52 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3438.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3310.0>] messages [] links [<5901.49.0>,<5901.3311.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3311.0> 2011-07-14 00:20:52 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3438.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:52 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 141 times in the past 139.656785 secs (last seen 0.996886 secs ago PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3447.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3448.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3449.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3450.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3451.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3452.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:52 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:52 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.3447.0> 2011-07-14 00:20:52 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3447.0> 2011-07-14 00:20:52 =============================================================================== ** Generic server <5901.3447.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.29987>}, init, {1310,617252,403399}, "default",#Port<5901.2811>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3447.0> 2011-07-14 00:20:52 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3447.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3310.0>] messages [] links [<5901.49.0>,<5901.3311.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3311.0> 2011-07-14 00:20:52 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3447.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3449.0> 2011-07-14 00:20:52 =============================================================================== ns_1@10.88.165.188:<5901.3449.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:52 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 144 times in the past 139.999791 secs (last seen 1.000179 secs ago PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3456.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3457.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3458.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3459.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3460.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3461.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3456.0> 2011-07-14 00:20:53 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3456.0> 2011-07-14 00:20:53 =============================================================================== ** Generic server <5901.3456.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.30063>}, init, {1310,617252,903745}, "default",#Port<5901.2812>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3456.0> 2011-07-14 00:20:53 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3456.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3310.0>] messages [] links [<5901.49.0>,<5901.3311.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3311.0> 2011-07-14 00:20:53 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3456.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:53 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 142 times in the past 140.659745 secs (last seen 1.00296 secs ago PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3466.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3467.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3468.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3469.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3470.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3471.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3466.0> 2011-07-14 00:20:53 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3466.0> 2011-07-14 00:20:53 =============================================================================== ** Generic server <5901.3466.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.30188>}, init, {1310,617253,406329}, "default",#Port<5901.2814>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3466.0> 2011-07-14 00:20:53 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3466.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3310.0>] messages [] links [<5901.49.0>,<5901.3311.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6671 SUPERVISOR REPORT <5901.3311.0> 2011-07-14 00:20:53 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3466.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3468.0> 2011-07-14 00:20:53 =============================================================================== ns_1@10.88.165.188:<5901.3468.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:53 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 145 times in the past 140.997512 secs (last seen 0.997721 secs ago PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3484.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3485.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3487.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3488.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3489.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3490.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:53 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:53 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.3484.0> 2011-07-14 00:20:53 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3484.0> 2011-07-14 00:20:53 =============================================================================== ** Generic server <5901.3484.0> terminating ** Last message in was list_vbuckets_prevstate ** When Server state == {state,{interval,#Ref<5901.0.0.30298>}, init, {1310,617253,900598}, "default",#Port<5901.2816>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3484.0> 2011-07-14 00:20:53 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3484.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3310.0>] messages [] links [<5901.49.0>,<5901.3311.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6701 SUPERVISOR REPORT <5901.3311.0> 2011-07-14 00:20:53 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3484.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker CRASH REPORT <5901.3481.0> 2011-07-14 00:20:53 =============================================================================== Crashing process initial_call {ns_janitor,cleanup,['Argument__1','Argument__2']} pid <5901.3481.0> registered_name [] error_info {exit,{{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, {gen_server,call, [{'ns_memcached-default','ns_1@10.88.165.188'}, list_vbuckets_prevstate,30000]}}, [{gen_server,call,3}, {ns_janitor,'-read_existing_map/4-lc$^0/1-0-',2}, {ns_janitor,read_existing_map,4}, {ns_janitor,cleanup,2}, {proc_lib,init_p_do_apply,3}]} ancestors [<5901.304.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<5901.52.0>] messages [] links [<5901.304.0>] dictionary [] trap_exit false status running heap_size 10946 stack_size 24 reductions 224 INFO REPORT <5901.227.0> 2011-07-14 00:20:53 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 146 times in the past 141.009124 secs (last seen 0.011612 secs ago INFO REPORT <5901.304.0> 2011-07-14 00:20:53 =============================================================================== ns_1@10.88.165.188:<5901.304.0>:ns_orchestrator:178: Janitor run exited for bucket "default" with reason {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, [{'ns_memcached-default', 'ns_1@10.88.165.188'}, list_vbuckets_prevstate, 30000]}} PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3492.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3493.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3494.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3495.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3496.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3497.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3492.0> 2011-07-14 00:20:54 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3492.0> 2011-07-14 00:20:54 =============================================================================== ** Generic server <5901.3492.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.30373>}, init, {1310,617253,911520}, "default",#Port<5901.2817>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3492.0> 2011-07-14 00:20:54 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3492.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3310.0>] messages [] links [<5901.49.0>,<5901.3311.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3311.0> 2011-07-14 00:20:54 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3492.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:54 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 143 times in the past 141.667768 secs (last seen 1.008023 secs ago PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3501.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3502.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3503.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3504.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3505.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3506.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3501.0> 2011-07-14 00:20:54 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3501.0> 2011-07-14 00:20:54 =============================================================================== ** Generic server <5901.3501.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.30455>}, init, {1310,617254,414390}, "default",#Port<5901.2818>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3501.0> 2011-07-14 00:20:54 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3501.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3310.0>] messages [] links [<5901.49.0>,<5901.3311.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3311.0> 2011-07-14 00:20:54 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3501.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3503.0> 2011-07-14 00:20:54 =============================================================================== ns_1@10.88.165.188:<5901.3503.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:54 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 147 times in the past 141.999879 secs (last seen 0.990755 secs ago PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3511.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3512.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3513.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3514.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3515.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3516.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:54 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:54 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.3511.0> 2011-07-14 00:20:55 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3511.0> 2011-07-14 00:20:55 =============================================================================== ** Generic server <5901.3511.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.30555>}, init, {1310,617254,902933}, "default",#Port<5901.2819>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3511.0> 2011-07-14 00:20:55 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3511.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3310.0>] messages [] links [<5901.49.0>,<5901.3311.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3311.0> 2011-07-14 00:20:55 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3511.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:55 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 144 times in the past 142.659796 secs (last seen 0.992028 secs ago PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3520.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3521.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3522.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3523.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3524.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3311.0> 2011-07-14 00:20:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3525.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3520.0> 2011-07-14 00:20:55 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3520.0> 2011-07-14 00:20:55 =============================================================================== ** Generic server <5901.3520.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.30637>}, init, {1310,617255,406455}, "default",#Port<5901.2820>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3520.0> 2011-07-14 00:20:55 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3520.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3310.0>] messages [] links [<5901.49.0>,<5901.3311.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3311.0> 2011-07-14 00:20:55 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3520.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker SUPERVISOR REPORT <5901.3311.0> 2011-07-14 00:20:55 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext shutdown reason reached_max_restart_intensity pid <5901.3520.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3522.0> 2011-07-14 00:20:55 =============================================================================== ns_1@10.88.165.188:<5901.3522.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:55 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 148 times in the past 142.999895 secs (last seen 1.000016 secs ago INFO REPORT <5901.3310.0> 2011-07-14 00:20:55 =============================================================================== ns_1@10.88.165.188:<5901.3310.0>:single_bucket_sup:24: per-bucket supervisor for "default" died with reason shutdown SUPERVISOR REPORT <5901.273.0> 2011-07-14 00:20:55 =============================================================================== Reporting supervisor {local,ns_bucket_sup} Child process errorContext child_terminated reason shutdown pid <5901.3310.0> name {per_bucket_sup,"default"} mfargs {single_bucket_sup,start_link,["default"]} restart_type permanent shutdown infinity child_type supervisor PROGRESS REPORT <5901.273.0> 2011-07-14 00:20:55 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<5901.3532.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3534.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3535.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3536.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3537.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3538.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3539.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3534.0> 2011-07-14 00:20:56 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3534.0> 2011-07-14 00:20:56 =============================================================================== ** Generic server <5901.3534.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.30725>}, init, {1310,617255,903237}, "default",#Port<5901.2821>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3534.0> 2011-07-14 00:20:56 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3534.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3532.0>] messages [] links [<5901.49.0>,<5901.3533.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3533.0> 2011-07-14 00:20:56 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3534.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:56 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 145 times in the past 143.659818 secs (last seen 1.000022 secs ago PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3544.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:56 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:56 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3545.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3546.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3547.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3548.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3549.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3544.0> 2011-07-14 00:20:56 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3544.0> 2011-07-14 00:20:56 =============================================================================== ** Generic server <5901.3544.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.30851>}, init, {1310,617256,406419}, "default",#Port<5901.2823>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3544.0> 2011-07-14 00:20:56 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3544.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3532.0>] messages [] links [<5901.49.0>,<5901.3533.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3533.0> 2011-07-14 00:20:56 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3544.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3546.0> 2011-07-14 00:20:56 =============================================================================== ns_1@10.88.165.188:<5901.3546.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:56 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 149 times in the past 144.000017 secs (last seen 1.000122 secs ago PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3553.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3554.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3555.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3556.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3557.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3558.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3553.0> 2011-07-14 00:20:57 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3553.0> 2011-07-14 00:20:57 =============================================================================== ** Generic server <5901.3553.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.30929>}, init, {1310,617256,903204}, "default",#Port<5901.2824>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3553.0> 2011-07-14 00:20:57 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3553.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3532.0>] messages [] links [<5901.49.0>,<5901.3533.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3533.0> 2011-07-14 00:20:57 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3553.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:57 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 146 times in the past 144.660029 secs (last seen 1.000211 secs ago PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3562.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3563.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3564.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:57 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:57 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3565.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3566.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3567.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3562.0> 2011-07-14 00:20:57 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3562.0> 2011-07-14 00:20:57 =============================================================================== ** Generic server <5901.3562.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.31015>}, init, {1310,617257,406602}, "default",#Port<5901.2825>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3562.0> 2011-07-14 00:20:57 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3562.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3532.0>] messages [] links [<5901.49.0>,<5901.3533.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3533.0> 2011-07-14 00:20:57 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3562.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3564.0> 2011-07-14 00:20:57 =============================================================================== ns_1@10.88.165.188:<5901.3564.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:57 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 150 times in the past 144.999983 secs (last seen 0.999966 secs ago PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3575.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3576.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3577.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3578.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3579.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3580.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3575.0> 2011-07-14 00:20:58 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3575.0> 2011-07-14 00:20:58 =============================================================================== ** Generic server <5901.3575.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.31116>}, init, {1310,617257,903106}, "default",#Port<5901.2826>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3575.0> 2011-07-14 00:20:58 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3575.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3532.0>] messages [] links [<5901.49.0>,<5901.3533.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3533.0> 2011-07-14 00:20:58 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3575.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:58 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 147 times in the past 145.659982 secs (last seen 0.999953 secs ago PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3584.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3585.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3586.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3587.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3588.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3589.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3584.0> 2011-07-14 00:20:58 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3584.0> 2011-07-14 00:20:58 =============================================================================== ** Generic server <5901.3584.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.31205>}, init, {1310,617258,406599}, "default",#Port<5901.2827>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3584.0> 2011-07-14 00:20:58 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3584.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3532.0>] messages [] links [<5901.49.0>,<5901.3533.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6665 SUPERVISOR REPORT <5901.3533.0> 2011-07-14 00:20:58 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3584.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3586.0> 2011-07-14 00:20:58 =============================================================================== ns_1@10.88.165.188:<5901.3586.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:58 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 151 times in the past 145.999994 secs (last seen 1.000011 secs ago PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3597.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:20:58 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:20:58 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3598.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3600.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3601.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3603.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3604.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3597.0> 2011-07-14 00:20:59 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3597.0> 2011-07-14 00:20:59 =============================================================================== ** Generic server <5901.3597.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.31378>}, init, {1310,617258,902938}, "default",#Port<5901.2829>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3597.0> 2011-07-14 00:20:59 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3597.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3532.0>] messages [] links [<5901.49.0>,<5901.3533.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.3533.0> 2011-07-14 00:20:59 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3597.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:20:59 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 148 times in the past 146.657211 secs (last seen 0.997229 secs ago PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3609.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3610.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3611.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3612.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3613.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3614.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3609.0> 2011-07-14 00:20:59 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3609.0> 2011-07-14 00:20:59 =============================================================================== ** Generic server <5901.3609.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.31505>}, init, {1310,617259,403796}, "default",#Port<5901.2831>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3609.0> 2011-07-14 00:20:59 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3609.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3532.0>] messages [] links [<5901.49.0>,<5901.3533.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3533.0> 2011-07-14 00:20:59 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3609.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3611.0> 2011-07-14 00:20:59 =============================================================================== ns_1@10.88.165.188:<5901.3611.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:20:59 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 152 times in the past 147.000177 secs (last seen 1.000183 secs ago PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3621.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3622.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3623.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3624.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3625.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:20:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3626.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3621.0> 2011-07-14 00:21:00 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3621.0> 2011-07-14 00:21:00 =============================================================================== ** Generic server <5901.3621.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.31592>}, init, {1310,617259,904578}, "default",#Port<5901.2832>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3621.0> 2011-07-14 00:21:00 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3621.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3532.0>] messages [] links [<5901.49.0>,<5901.3533.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3533.0> 2011-07-14 00:21:00 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3621.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:00 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 149 times in the past 147.657393 secs (last seen 1.000182 secs ago PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3630.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3631.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:00 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:00 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3632.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3633.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3634.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3635.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3630.0> 2011-07-14 00:21:00 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3630.0> 2011-07-14 00:21:00 =============================================================================== ** Generic server <5901.3630.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.31679>}, init, {1310,617260,403963}, "default",#Port<5901.2833>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3630.0> 2011-07-14 00:21:00 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3630.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3532.0>] messages [] links [<5901.49.0>,<5901.3533.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3533.0> 2011-07-14 00:21:00 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3630.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3632.0> 2011-07-14 00:21:00 =============================================================================== ns_1@10.88.165.188:<5901.3632.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:00 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 153 times in the past 147.997513 secs (last seen 0.997336 secs ago PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3639.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3640.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3641.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3642.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3643.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3644.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3639.0> 2011-07-14 00:21:01 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3639.0> 2011-07-14 00:21:01 =============================================================================== ** Generic server <5901.3639.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.31755>}, init, {1310,617260,900556}, "default",#Port<5901.2834>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3639.0> 2011-07-14 00:21:01 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3639.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3532.0>] messages [] links [<5901.49.0>,<5901.3533.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3533.0> 2011-07-14 00:21:01 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3639.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:01 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 150 times in the past 148.656285 secs (last seen 0.998892 secs ago PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3649.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3650.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3651.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3652.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3653.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3654.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3649.0> 2011-07-14 00:21:01 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3649.0> 2011-07-14 00:21:01 =============================================================================== ** Generic server <5901.3649.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.31863>}, init, {1310,617261,402895}, "default",#Port<5901.2835>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3649.0> 2011-07-14 00:21:01 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3649.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3532.0>] messages [] links [<5901.49.0>,<5901.3533.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3533.0> 2011-07-14 00:21:01 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3649.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3651.0> 2011-07-14 00:21:01 =============================================================================== ns_1@10.88.165.188:<5901.3651.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:01 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 154 times in the past 149.00035 secs (last seen 1.002837 secs ago PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3661.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:01 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:01 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3662.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3663.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3664.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3665.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3666.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3661.0> 2011-07-14 00:21:02 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3661.0> 2011-07-14 00:21:02 =============================================================================== ** Generic server <5901.3661.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.31944>}, init, {1310,617261,903385}, "default",#Port<5901.2836>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3661.0> 2011-07-14 00:21:02 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3661.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3532.0>] messages [] links [<5901.49.0>,<5901.3533.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3533.0> 2011-07-14 00:21:02 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3661.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:02 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 151 times in the past 149.658062 secs (last seen 1.001777 secs ago PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3671.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3672.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3673.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3674.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3675.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3676.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3671.0> 2011-07-14 00:21:02 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3671.0> 2011-07-14 00:21:02 =============================================================================== ** Generic server <5901.3671.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.32077>}, init, {1310,617262,406001}, "default",#Port<5901.2838>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3671.0> 2011-07-14 00:21:02 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3671.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3532.0>] messages [] links [<5901.49.0>,<5901.3533.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3533.0> 2011-07-14 00:21:02 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3671.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3673.0> 2011-07-14 00:21:02 =============================================================================== ns_1@10.88.165.188:<5901.3673.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:02 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 155 times in the past 149.997492 secs (last seen 0.997142 secs ago PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3680.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3681.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3682.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3683.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3684.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3685.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:02 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:02 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.3680.0> 2011-07-14 00:21:03 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3680.0> 2011-07-14 00:21:03 =============================================================================== ** Generic server <5901.3680.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.32162>}, init, {1310,617262,900703}, "default",#Port<5901.2839>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3680.0> 2011-07-14 00:21:03 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3680.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3532.0>] messages [] links [<5901.49.0>,<5901.3533.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3533.0> 2011-07-14 00:21:03 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3680.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:03 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 152 times in the past 150.656327 secs (last seen 0.998265 secs ago PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3689.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3690.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3691.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3692.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3693.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3694.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3689.0> 2011-07-14 00:21:03 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3689.0> 2011-07-14 00:21:03 =============================================================================== ** Generic server <5901.3689.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.32244>}, init, {1310,617263,403134}, "default",#Port<5901.2840>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3689.0> 2011-07-14 00:21:03 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3689.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3532.0>] messages [{'$gen_call',{<5901.3706.0>, {#Ref<5901.0.0.32345>,'ns_1@10.88.165.188'}}, connected}] links [<5901.49.0>,<5901.3533.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6665 SUPERVISOR REPORT <5901.3533.0> 2011-07-14 00:21:03 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3689.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3691.0> 2011-07-14 00:21:03 =============================================================================== ns_1@10.88.165.188:<5901.3691.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:03 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 156 times in the past 150.997931 secs (last seen 1.000439 secs ago INFO REPORT <5901.3704.0> 2011-07-14 00:21:03 =============================================================================== ns_1@10.88.165.188:<5901.3704.0>:ns_janitor:302: Waiting for "default" on ['ns_1@10.88.165.188'] SUPERVISOR REPORT <5901.258.0> 2011-07-14 00:21:03 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call,['ns_memcached-default',topkeys,30000]}} pid <5901.1392.0> name hot_keys_keeper mfargs {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <5901.258.0> 2011-07-14 00:21:03 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<5901.3709.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3707.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3710.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3711.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3712.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3713.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3714.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3707.0> 2011-07-14 00:21:04 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3707.0> 2011-07-14 00:21:04 =============================================================================== ** Generic server <5901.3707.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.32360>}, init, {1310,617263,921655}, "default",#Port<5901.2842>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3707.0> 2011-07-14 00:21:04 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3707.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3532.0>] messages [] links [<5901.49.0>,<5901.3533.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3533.0> 2011-07-14 00:21:04 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3707.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:04 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 153 times in the past 151.656331 secs (last seen 1.000004 secs ago PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3719.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3720.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:04 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:04 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3721.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3722.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3723.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3724.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3719.0> 2011-07-14 00:21:04 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3719.0> 2011-07-14 00:21:04 =============================================================================== ** Generic server <5901.3719.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.32472>}, init, {1310,617264,402905}, "default",#Port<5901.2845>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3719.0> 2011-07-14 00:21:04 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3719.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3532.0>] messages [] links [<5901.49.0>,<5901.3533.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3533.0> 2011-07-14 00:21:04 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3719.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3721.0> 2011-07-14 00:21:04 =============================================================================== ns_1@10.88.165.188:<5901.3721.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:04 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 157 times in the past 151.997509 secs (last seen 0.999578 secs ago PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3728.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3729.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3731.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3732.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3733.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3734.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3704.0> 2011-07-14 00:21:04 =============================================================================== ns_1@10.88.165.188:<5901.3704.0>:ns_janitor:302: Waiting for "default" on ['ns_1@10.88.23.62'] INFO REPORT <5901.3728.0> 2011-07-14 00:21:05 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3728.0> 2011-07-14 00:21:05 =============================================================================== ** Generic server <5901.3728.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.32548>}, init, {1310,617264,900570}, "default",#Port<5901.2846>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3728.0> 2011-07-14 00:21:05 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3728.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3532.0>] messages [] links [<5901.49.0>,<5901.3533.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7210 SUPERVISOR REPORT <5901.3533.0> 2011-07-14 00:21:05 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3728.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:05 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 154 times in the past 152.656531 secs (last seen 1.0002 secs ago PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3739.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3740.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3741.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3742.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3743.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3744.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3739.0> 2011-07-14 00:21:05 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3739.0> 2011-07-14 00:21:05 =============================================================================== ** Generic server <5901.3739.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.32678>}, init, {1310,617265,404755}, "default",#Port<5901.2848>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3739.0> 2011-07-14 00:21:05 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3739.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3532.0>] messages [] links [<5901.49.0>,<5901.3533.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3533.0> 2011-07-14 00:21:05 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3739.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3741.0> 2011-07-14 00:21:05 =============================================================================== ns_1@10.88.165.188:<5901.3741.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:05 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 158 times in the past 152.997374 secs (last seen 0.999865 secs ago PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3751.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3752.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3753.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:05 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:05 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3754.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3755.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3533.0> 2011-07-14 00:21:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3756.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3704.0> 2011-07-14 00:21:05 =============================================================================== ns_1@10.88.165.188:<5901.3704.0>:ns_janitor:302: Waiting for "default" on ['ns_1@10.88.23.62'] INFO REPORT <5901.3751.0> 2011-07-14 00:21:06 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3751.0> 2011-07-14 00:21:06 =============================================================================== ** Generic server <5901.3751.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.32759>}, init, {1310,617265,900375}, "default",#Port<5901.2849>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3751.0> 2011-07-14 00:21:06 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3751.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3532.0>] messages [] links [<5901.49.0>,<5901.3533.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7210 SUPERVISOR REPORT <5901.3533.0> 2011-07-14 00:21:06 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3751.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker SUPERVISOR REPORT <5901.3533.0> 2011-07-14 00:21:06 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext shutdown reason reached_max_restart_intensity pid <5901.3751.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:06 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 155 times in the past 153.65651 secs (last seen 0.999979 secs ago INFO REPORT <5901.3532.0> 2011-07-14 00:21:06 =============================================================================== ns_1@10.88.165.188:<5901.3532.0>:single_bucket_sup:24: per-bucket supervisor for "default" died with reason shutdown SUPERVISOR REPORT <5901.273.0> 2011-07-14 00:21:06 =============================================================================== Reporting supervisor {local,ns_bucket_sup} Child process errorContext child_terminated reason shutdown pid <5901.3532.0> name {per_bucket_sup,"default"} mfargs {single_bucket_sup,start_link,["default"]} restart_type permanent shutdown infinity child_type supervisor PROGRESS REPORT <5901.273.0> 2011-07-14 00:21:06 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<5901.3761.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3763.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3764.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3765.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3766.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3767.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3768.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3763.0> 2011-07-14 00:21:06 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3763.0> 2011-07-14 00:21:06 =============================================================================== ** Generic server <5901.3763.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.32851>}, init, {1310,617266,403434}, "default",#Port<5901.2850>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3763.0> 2011-07-14 00:21:06 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3763.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3761.0>] messages [] links [<5901.49.0>,<5901.3762.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3762.0> 2011-07-14 00:21:06 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3763.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3765.0> 2011-07-14 00:21:06 =============================================================================== ns_1@10.88.165.188:<5901.3765.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:06 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 159 times in the past 153.998411 secs (last seen 1.001037 secs ago PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3772.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3774.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3775.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3704.0> 2011-07-14 00:21:06 =============================================================================== ns_1@10.88.165.188:<5901.3704.0>:ns_janitor:302: Waiting for "default" on ['ns_1@10.88.165.188'] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3776.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3777.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3778.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3772.0> 2011-07-14 00:21:07 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3772.0> 2011-07-14 00:21:07 =============================================================================== ** Generic server <5901.3772.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.32934>}, init, {1310,617266,905563}, "default",#Port<5901.2851>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3772.0> 2011-07-14 00:21:07 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3772.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3761.0>] messages [] links [<5901.49.0>,<5901.3762.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3762.0> 2011-07-14 00:21:07 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3772.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:07 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 156 times in the past 154.656517 secs (last seen 1.000007 secs ago PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3783.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3784.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3785.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:07 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:07 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3786.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3787.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3788.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3783.0> 2011-07-14 00:21:07 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3783.0> 2011-07-14 00:21:07 =============================================================================== ** Generic server <5901.3783.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.33042>}, init, {1310,617267,403136}, "default",#Port<5901.2852>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3783.0> 2011-07-14 00:21:07 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3783.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3761.0>] messages [] links [<5901.49.0>,<5901.3762.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3762.0> 2011-07-14 00:21:07 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3783.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3785.0> 2011-07-14 00:21:07 =============================================================================== ns_1@10.88.165.188:<5901.3785.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:07 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 160 times in the past 154.997353 secs (last seen 0.998942 secs ago PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3795.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3796.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3797.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3798.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3799.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3800.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3795.0> 2011-07-14 00:21:07 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3795.0> 2011-07-14 00:21:07 =============================================================================== ** Generic server <5901.3795.0> terminating ** Last message in was list_vbuckets_prevstate ** When Server state == {state,{interval,#Ref<5901.0.0.33128>}, init, {1310,617267,900394}, "default",#Port<5901.2854>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3795.0> 2011-07-14 00:21:07 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3795.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3761.0>] messages [] links [<5901.49.0>,<5901.3762.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3762.0> 2011-07-14 00:21:07 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3795.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker CRASH REPORT <5901.3704.0> 2011-07-14 00:21:07 =============================================================================== Crashing process initial_call {ns_janitor,cleanup,['Argument__1','Argument__2']} pid <5901.3704.0> registered_name [] error_info {exit,{{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, {gen_server,call, [{'ns_memcached-default','ns_1@10.88.165.188'}, list_vbuckets_prevstate,30000]}}, [{gen_server,call,3}, {ns_janitor,'-read_existing_map/4-lc$^0/1-0-',2}, {ns_janitor,read_existing_map,4}, {ns_janitor,cleanup,2}, {proc_lib,init_p_do_apply,3}]} ancestors [<5901.304.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<5901.52.0>] messages [] links [<5901.304.0>] dictionary [] trap_exit false status running heap_size 10946 stack_size 24 reductions 374 INFO REPORT <5901.227.0> 2011-07-14 00:21:07 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 161 times in the past 155.016943 secs (last seen 0.01959 secs ago INFO REPORT <5901.304.0> 2011-07-14 00:21:07 =============================================================================== ns_1@10.88.165.188:<5901.304.0>:ns_orchestrator:178: Janitor run exited for bucket "default" with reason {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, [{'ns_memcached-default', 'ns_1@10.88.165.188'}, list_vbuckets_prevstate, 30000]}} PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3803.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3804.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3805.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3806.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3807.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3808.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3803.0> 2011-07-14 00:21:08 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3803.0> 2011-07-14 00:21:08 =============================================================================== ** Generic server <5901.3803.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.33205>}, init, {1310,617267,919139}, "default",#Port<5901.2855>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3803.0> 2011-07-14 00:21:08 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3803.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3761.0>] messages [] links [<5901.49.0>,<5901.3762.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3762.0> 2011-07-14 00:21:08 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3803.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:08 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 157 times in the past 155.676703 secs (last seen 1.020186 secs ago PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3812.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3813.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3814.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3815.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3816.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3817.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3812.0> 2011-07-14 00:21:08 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3812.0> 2011-07-14 00:21:08 =============================================================================== ** Generic server <5901.3812.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.33317>}, init, {1310,617268,423237}, "default",#Port<5901.2857>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3812.0> 2011-07-14 00:21:08 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3812.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3761.0>] messages [] links [<5901.49.0>,<5901.3762.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6665 SUPERVISOR REPORT <5901.3762.0> 2011-07-14 00:21:08 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3812.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3814.0> 2011-07-14 00:21:08 =============================================================================== ns_1@10.88.165.188:<5901.3814.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:08 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 162 times in the past 155.997344 secs (last seen 0.980401 secs ago PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3824.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:08 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:08 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3825.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3827.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3828.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3830.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3831.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3824.0> 2011-07-14 00:21:09 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3824.0> 2011-07-14 00:21:09 =============================================================================== ** Generic server <5901.3824.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.33409>}, init, {1310,617268,900290}, "default",#Port<5901.2858>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3824.0> 2011-07-14 00:21:09 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3824.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3761.0>] messages [] links [<5901.49.0>,<5901.3762.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.3762.0> 2011-07-14 00:21:09 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3824.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:09 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 158 times in the past 156.658199 secs (last seen 0.981496 secs ago PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3835.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3836.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3837.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3838.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3839.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3840.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3835.0> 2011-07-14 00:21:09 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3835.0> 2011-07-14 00:21:09 =============================================================================== ** Generic server <5901.3835.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.33507>}, init, {1310,617269,404879}, "default",#Port<5901.2859>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3835.0> 2011-07-14 00:21:09 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3835.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3761.0>] messages [] links [<5901.49.0>,<5901.3762.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3762.0> 2011-07-14 00:21:09 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3835.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3837.0> 2011-07-14 00:21:09 =============================================================================== ns_1@10.88.165.188:<5901.3837.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:09 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 163 times in the past 157.00076 secs (last seen 1.003416 secs ago PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3847.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:09 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:09 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3848.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3849.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3850.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3851.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3852.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3847.0> 2011-07-14 00:21:10 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3847.0> 2011-07-14 00:21:10 =============================================================================== ** Generic server <5901.3847.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.33588>}, init, {1310,617269,903785}, "default",#Port<5901.2860>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3847.0> 2011-07-14 00:21:10 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3847.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3761.0>] messages [] links [<5901.49.0>,<5901.3762.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3762.0> 2011-07-14 00:21:10 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3847.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:10 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 159 times in the past 157.66063 secs (last seen 1.002431 secs ago PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3857.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3858.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3859.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3860.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3861.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3862.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3857.0> 2011-07-14 00:21:10 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3857.0> 2011-07-14 00:21:10 =============================================================================== ** Generic server <5901.3857.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.33696>}, init, {1310,617270,407161}, "default",#Port<5901.2861>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3857.0> 2011-07-14 00:21:10 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3857.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3761.0>] messages [] links [<5901.49.0>,<5901.3762.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3762.0> 2011-07-14 00:21:10 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3857.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3859.0> 2011-07-14 00:21:10 =============================================================================== ns_1@10.88.165.188:<5901.3859.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:10 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 164 times in the past 158.001021 secs (last seen 1.000261 secs ago INFO REPORT <5901.269.0> 2011-07-14 00:21:10 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:10 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3866.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3867.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3868.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3869.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3870.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3871.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3866.0> 2011-07-14 00:21:11 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3866.0> 2011-07-14 00:21:11 =============================================================================== ** Generic server <5901.3866.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.33776>}, init, {1310,617270,904211}, "default",#Port<5901.2862>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3866.0> 2011-07-14 00:21:11 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3866.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3761.0>] messages [] links [<5901.49.0>,<5901.3762.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3762.0> 2011-07-14 00:21:11 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3866.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:11 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 160 times in the past 158.66086 secs (last seen 1.00023 secs ago PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3876.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3877.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3878.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3879.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3880.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3881.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3876.0> 2011-07-14 00:21:11 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3876.0> 2011-07-14 00:21:11 =============================================================================== ** Generic server <5901.3876.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.33895>}, init, {1310,617271,407487}, "default",#Port<5901.2864>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3876.0> 2011-07-14 00:21:11 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3876.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3761.0>] messages [] links [<5901.49.0>,<5901.3762.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3762.0> 2011-07-14 00:21:11 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3876.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3878.0> 2011-07-14 00:21:11 =============================================================================== ns_1@10.88.165.188:<5901.3878.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:11 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 165 times in the past 159.001024 secs (last seen 1.000003 secs ago PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3888.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3889.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3890.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3891.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3892.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3893.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3888.0> 2011-07-14 00:21:12 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3888.0> 2011-07-14 00:21:12 =============================================================================== ** Generic server <5901.3888.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.33982>}, init, {1310,617271,904265}, "default",#Port<5901.2865>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3888.0> 2011-07-14 00:21:12 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3888.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3761.0>] messages [] links [<5901.49.0>,<5901.3762.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3762.0> 2011-07-14 00:21:12 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3888.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:12 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 161 times in the past 159.660841 secs (last seen 0.999981 secs ago PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3897.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3898.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3899.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:12 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:12 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3900.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3901.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3902.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3897.0> 2011-07-14 00:21:12 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3897.0> 2011-07-14 00:21:12 =============================================================================== ** Generic server <5901.3897.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.34068>}, init, {1310,617272,407439}, "default",#Port<5901.2866>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3897.0> 2011-07-14 00:21:12 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3897.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3761.0>] messages [] links [<5901.49.0>,<5901.3762.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3762.0> 2011-07-14 00:21:12 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3897.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3899.0> 2011-07-14 00:21:12 =============================================================================== ns_1@10.88.165.188:<5901.3899.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:12 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 166 times in the past 159.997439 secs (last seen 0.996415 secs ago PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3906.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3907.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3908.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3909.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3910.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3911.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3906.0> 2011-07-14 00:21:13 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3906.0> 2011-07-14 00:21:13 =============================================================================== ** Generic server <5901.3906.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.34148>}, init, {1310,617272,900657}, "default",#Port<5901.2867>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3906.0> 2011-07-14 00:21:13 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3906.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3761.0>] messages [] links [<5901.49.0>,<5901.3762.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3762.0> 2011-07-14 00:21:13 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3906.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:13 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 162 times in the past 160.656898 secs (last seen 0.996057 secs ago PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3928.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3929.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3930.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3931.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3932.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3933.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.256.0> 2011-07-14 00:21:13 =============================================================================== ns_1@10.88.165.188:<5901.256.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.88.165.188', [{last_heard,{1310,617273,600642}}, {active_buckets,["default"]}, {ready_buckets,[]}, {replication,[{"default",1.0}]}, {memory, [{total,28805368}, {processes,7041856}, {processes_used,7033072}, {system,21763512}, {atom,814593}, {atom_used,805810}, {binary,501216}, {code,7689836}, {ets,3446888}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,181}, {memory_data,{644874240,583725056,{<5901.80.0>,601112}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 60080 kB\nBuffers: 22208 kB\nCached: 366824 kB\nSwapCached: 0 kB\nActive: 222256 kB\nInactive: 270140 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 60080 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 672 kB\nWriteback: 0 kB\nAnonPages: 103348 kB\nMapped: 12304 kB\nSlab: 36964 kB\nPageTables: 4164 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 552372 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,375627776}, {buffered_memory,22740992}, {free_memory,61521920}, {total_memory,644874240}]}, {statistics, [{wall_clock,{166634,1}}, {context_switches,{88031,0}}, {garbage_collection,{18729,38848778,0}}, {io,{{input,16200025},{output,8545204}}}, {reductions,{10357987,877291}}, {run_queue,0}, {runtime,{650,10}}]}]}, {'ns_1@10.88.23.62', [{last_heard,{1310,617273,607111}}, {active_buckets,["default"]}, {ready_buckets,[]}, {replication,[{"default",1.0}]}, {memory, [{total,30069264}, {processes,7826840}, {processes_used,7799296}, {system,22242424}, {atom,820249}, {atom_used,814132}, {binary,908136}, {code,7741308}, {ets,3460080}]}, {system_stats,[{cpu_utilization_rate,1.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,185}, {memory_data,{644874240,588296192,{<5990.256.0>,830584}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 55004 kB\nBuffers: 21788 kB\nCached: 368696 kB\nSwapCached: 0 kB\nActive: 227816 kB\nInactive: 269496 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 55004 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 272 kB\nWriteback: 0 kB\nAnonPages: 106876 kB\nMapped: 12316 kB\nSlab: 37112 kB\nPageTables: 4116 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 554696 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,377544704}, {buffered_memory,22310912}, {free_memory,56324096}, {total_memory,644874240}]}, {statistics, [{wall_clock,{181505,1}}, {context_switches,{104370,0}}, {garbage_collection,{21275,58803442,0}}, {io,{{input,17295616},{output,9109580}}}, {reductions,{12517584,950553}}, {run_queue,0}, {runtime,{1040,40}}]}]}] INFO REPORT <5901.3928.0> 2011-07-14 00:21:13 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3928.0> 2011-07-14 00:21:13 =============================================================================== ** Generic server <5901.3928.0> terminating ** Last message in was list_vbuckets_prevstate ** When Server state == {state,{interval,#Ref<5901.0.0.34310>}, init, {1310,617273,403479}, "default",#Port<5901.2887>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3928.0> 2011-07-14 00:21:13 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3928.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3761.0>] messages [{'$gen_call',{<5901.3930.0>,#Ref<5901.0.0.34436>},{stats,<<>>}}] links [<5901.49.0>,<5901.3762.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6671 SUPERVISOR REPORT <5901.3762.0> 2011-07-14 00:21:13 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3928.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker CRASH REPORT <5901.3942.0> 2011-07-14 00:21:13 =============================================================================== Crashing process initial_call {ns_janitor,cleanup,['Argument__1','Argument__2']} pid <5901.3942.0> registered_name [] error_info {exit,{{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, {gen_server,call, [{'ns_memcached-default','ns_1@10.88.165.188'}, list_vbuckets_prevstate,30000]}}, [{gen_server,call,3}, {ns_janitor,'-read_existing_map/4-lc$^0/1-0-',2}, {ns_janitor,read_existing_map,4}, {ns_janitor,cleanup,2}, {proc_lib,init_p_do_apply,3}]} ancestors [<5901.304.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<5901.52.0>] messages [] links [<5901.304.0>] dictionary [] trap_exit false status running heap_size 10946 stack_size 24 reductions 224 ERROR REPORT <5901.3930.0> 2011-07-14 00:21:13 =============================================================================== ns_1@10.88.165.188:<5901.3930.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:13 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 167 times in the past 160.997954 secs (last seen 1.000515 secs ago INFO REPORT <5901.304.0> 2011-07-14 00:21:13 =============================================================================== ns_1@10.88.165.188:<5901.304.0>:ns_orchestrator:178: Janitor run exited for bucket "default" with reason {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, [{'ns_memcached-default', 'ns_1@10.88.165.188'}, list_vbuckets_prevstate, 30000]}} SUPERVISOR REPORT <5901.258.0> 2011-07-14 00:21:13 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call,['ns_memcached-default',topkeys,30000]}} pid <5901.3709.0> name hot_keys_keeper mfargs {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <5901.258.0> 2011-07-14 00:21:13 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<5901.3951.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:13 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:13 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3948.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3952.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] SUPERVISOR REPORT <5901.258.0> 2011-07-14 00:21:13 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.88.165.188'}, {latest,minute,1}]}} pid <5901.3272.0> name menelaus_web_alerts_srv mfargs {menelaus_web_alerts_srv,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3954.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.258.0> 2011-07-14 00:21:13 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<5901.3955.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3956.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3957.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3958.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3948.0> 2011-07-14 00:21:14 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3948.0> 2011-07-14 00:21:14 =============================================================================== ** Generic server <5901.3948.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.34445>}, init, {1310,617273,912130}, "default",#Port<5901.2890>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3948.0> 2011-07-14 00:21:14 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3948.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3761.0>] messages [] links [<5901.49.0>,<5901.3762.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3762.0> 2011-07-14 00:21:14 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3948.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:14 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 163 times in the past 161.657028 secs (last seen 1.00013 secs ago PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3962.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3963.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3964.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3965.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3966.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3967.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3962.0> 2011-07-14 00:21:14 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3962.0> 2011-07-14 00:21:14 =============================================================================== ** Generic server <5901.3962.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.34624>}, init, {1310,617274,403701}, "default",#Port<5901.2891>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3962.0> 2011-07-14 00:21:14 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3962.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3761.0>] messages [] links [<5901.49.0>,<5901.3762.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3762.0> 2011-07-14 00:21:14 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3962.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3964.0> 2011-07-14 00:21:14 =============================================================================== ns_1@10.88.165.188:<5901.3964.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:14 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 168 times in the past 161.997419 secs (last seen 0.999465 secs ago PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3971.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3972.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3973.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3974.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3975.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3976.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:14 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:14 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.3971.0> 2011-07-14 00:21:15 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3971.0> 2011-07-14 00:21:15 =============================================================================== ** Generic server <5901.3971.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.34704>}, init, {1310,617274,900633}, "default",#Port<5901.2892>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3971.0> 2011-07-14 00:21:15 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3971.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3761.0>] messages [] links [<5901.49.0>,<5901.3762.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3762.0> 2011-07-14 00:21:15 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3971.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:15 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 164 times in the past 162.663872 secs (last seen 1.006844 secs ago PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3980.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3981.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3982.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3983.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3984.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3985.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3980.0> 2011-07-14 00:21:15 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3980.0> 2011-07-14 00:21:15 =============================================================================== ** Generic server <5901.3980.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.34786>}, init, {1310,617275,410503}, "default",#Port<5901.2893>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3980.0> 2011-07-14 00:21:15 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3980.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3761.0>] messages [] links [<5901.49.0>,<5901.3762.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.3762.0> 2011-07-14 00:21:15 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3980.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.3982.0> 2011-07-14 00:21:15 =============================================================================== ns_1@10.88.165.188:<5901.3982.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:15 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 169 times in the past 163.001117 secs (last seen 1.003698 secs ago PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3992.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3993.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3994.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3995.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3996.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.3762.0> 2011-07-14 00:21:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.3997.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.3992.0> 2011-07-14 00:21:16 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.3992.0> 2011-07-14 00:21:16 =============================================================================== ** Generic server <5901.3992.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.34874>}, init, {1310,617275,904095}, "default",#Port<5901.2894>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.3992.0> 2011-07-14 00:21:16 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.3992.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.3761.0>] messages [] links [<5901.49.0>,<5901.3762.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.3762.0> 2011-07-14 00:21:16 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.3992.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker SUPERVISOR REPORT <5901.3762.0> 2011-07-14 00:21:16 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext shutdown reason reached_max_restart_intensity pid <5901.3992.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:16 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 165 times in the past 163.658214 secs (last seen 0.994342 secs ago INFO REPORT <5901.3761.0> 2011-07-14 00:21:16 =============================================================================== ns_1@10.88.165.188:<5901.3761.0>:single_bucket_sup:24: per-bucket supervisor for "default" died with reason shutdown SUPERVISOR REPORT <5901.273.0> 2011-07-14 00:21:16 =============================================================================== Reporting supervisor {local,ns_bucket_sup} Child process errorContext child_terminated reason shutdown pid <5901.3761.0> name {per_bucket_sup,"default"} mfargs {single_bucket_sup,start_link,["default"]} restart_type permanent shutdown infinity child_type supervisor PROGRESS REPORT <5901.273.0> 2011-07-14 00:21:16 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<5901.4001.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4003.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4004.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4005.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4006.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4007.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4008.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:16 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:16 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.4003.0> 2011-07-14 00:21:16 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4003.0> 2011-07-14 00:21:16 =============================================================================== ** Generic server <5901.4003.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.34960>}, init, {1310,617276,405019}, "default",#Port<5901.2895>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4003.0> 2011-07-14 00:21:16 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4003.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4001.0>] messages [] links [<5901.49.0>,<5901.4002.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4002.0> 2011-07-14 00:21:16 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4003.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4005.0> 2011-07-14 00:21:16 =============================================================================== ns_1@10.88.165.188:<5901.4005.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:16 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 170 times in the past 163.99735 secs (last seen 0.996233 secs ago PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4013.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4014.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4015.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4016.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4017.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4018.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4013.0> 2011-07-14 00:21:17 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4013.0> 2011-07-14 00:21:17 =============================================================================== ** Generic server <5901.4013.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.35056>}, init, {1310,617276,900441}, "default",#Port<5901.2896>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4013.0> 2011-07-14 00:21:17 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4013.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4001.0>] messages [] links [<5901.49.0>,<5901.4002.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4002.0> 2011-07-14 00:21:17 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4013.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:17 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 166 times in the past 164.654576 secs (last seen 0.996362 secs ago PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4023.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4024.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4025.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4026.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4027.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4028.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4023.0> 2011-07-14 00:21:17 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4023.0> 2011-07-14 00:21:17 =============================================================================== ** Generic server <5901.4023.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.35180>}, init, {1310,617277,401107}, "default",#Port<5901.2898>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4023.0> 2011-07-14 00:21:17 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4023.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4001.0>] messages [] links [<5901.49.0>,<5901.4002.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4002.0> 2011-07-14 00:21:17 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4023.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4025.0> 2011-07-14 00:21:17 =============================================================================== ns_1@10.88.165.188:<5901.4025.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:17 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 171 times in the past 164.998228 secs (last seen 1.000878 secs ago PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4035.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:17 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:17 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4036.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4037.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4038.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4039.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4040.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4035.0> 2011-07-14 00:21:18 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4035.0> 2011-07-14 00:21:18 =============================================================================== ** Generic server <5901.4035.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.35261>}, init, {1310,617277,901198}, "default",#Port<5901.2899>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4035.0> 2011-07-14 00:21:18 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4035.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4001.0>] messages [] links [<5901.49.0>,<5901.4002.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4002.0> 2011-07-14 00:21:18 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4035.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:18 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 167 times in the past 165.657208 secs (last seen 1.002632 secs ago PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4044.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4045.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4046.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4047.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4048.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4049.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4044.0> 2011-07-14 00:21:18 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4044.0> 2011-07-14 00:21:18 =============================================================================== ** Generic server <5901.4044.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.35350>}, init, {1310,617278,403822}, "default",#Port<5901.2900>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4044.0> 2011-07-14 00:21:18 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4044.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4001.0>] messages [] links [<5901.49.0>,<5901.4002.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6665 SUPERVISOR REPORT <5901.4002.0> 2011-07-14 00:21:18 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4044.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4046.0> 2011-07-14 00:21:18 =============================================================================== ns_1@10.88.165.188:<5901.4046.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:18 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 172 times in the past 165.998264 secs (last seen 1.000036 secs ago PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4056.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:18 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:18 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4057.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4059.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4060.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4062.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4063.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4056.0> 2011-07-14 00:21:19 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4056.0> 2011-07-14 00:21:19 =============================================================================== ** Generic server <5901.4056.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.35446>}, init, {1310,617278,901778}, "default",#Port<5901.2901>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4056.0> 2011-07-14 00:21:19 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4056.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4001.0>] messages [] links [<5901.49.0>,<5901.4002.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.4002.0> 2011-07-14 00:21:19 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4056.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:19 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 168 times in the past 166.656145 secs (last seen 0.998937 secs ago PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4067.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4068.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4069.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4070.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4071.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4072.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4067.0> 2011-07-14 00:21:19 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4067.0> 2011-07-14 00:21:19 =============================================================================== ** Generic server <5901.4067.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.35539>}, init, {1310,617279,402730}, "default",#Port<5901.2902>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4067.0> 2011-07-14 00:21:19 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4067.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4001.0>] messages [check_started] links [<5901.49.0>,<5901.4002.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4002.0> 2011-07-14 00:21:19 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4067.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4069.0> 2011-07-14 00:21:19 =============================================================================== ns_1@10.88.165.188:<5901.4069.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:19 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 173 times in the past 167.002064 secs (last seen 1.0038 secs ago PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4080.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:19 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:19 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4081.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4082.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4083.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4084.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4085.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.239.0> 2011-07-14 00:21:20 =============================================================================== ns_1@10.88.165.188:<5901.239.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.4080.0> 2011-07-14 00:21:20 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4080.0> 2011-07-14 00:21:20 =============================================================================== ** Generic server <5901.4080.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.35644>}, init, {1310,617279,905065}, "default",#Port<5901.2903>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4080.0> 2011-07-14 00:21:20 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4080.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4001.0>] messages [] links [<5901.49.0>,<5901.4002.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4002.0> 2011-07-14 00:21:20 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4080.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:20 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 169 times in the past 167.660156 secs (last seen 1.004011 secs ago PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4092.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4093.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4094.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4095.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4096.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4097.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4092.0> 2011-07-14 00:21:20 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4092.0> 2011-07-14 00:21:20 =============================================================================== ** Generic server <5901.4092.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.35782>}, init, {1310,617280,406766}, "default",#Port<5901.2906>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4092.0> 2011-07-14 00:21:20 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4092.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4001.0>] messages [] links [<5901.49.0>,<5901.4002.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4002.0> 2011-07-14 00:21:20 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4092.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4094.0> 2011-07-14 00:21:20 =============================================================================== ns_1@10.88.165.188:<5901.4094.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:20 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 174 times in the past 167.997995 secs (last seen 0.995931 secs ago PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4101.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4102.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4103.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4104.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4105.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4106.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:20 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:20 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.4101.0> 2011-07-14 00:21:21 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4101.0> 2011-07-14 00:21:21 =============================================================================== ** Generic server <5901.4101.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.35858>}, init, {1310,617280,903350}, "default",#Port<5901.2907>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4101.0> 2011-07-14 00:21:21 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4101.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4001.0>] messages [] links [<5901.49.0>,<5901.4002.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4002.0> 2011-07-14 00:21:21 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4101.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:21 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 170 times in the past 168.656298 secs (last seen 0.996142 secs ago PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4110.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4111.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4112.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4113.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4114.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4115.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4110.0> 2011-07-14 00:21:21 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4110.0> 2011-07-14 00:21:21 =============================================================================== ** Generic server <5901.4110.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.35944>}, init, {1310,617281,402935}, "default",#Port<5901.2908>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4110.0> 2011-07-14 00:21:21 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4110.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4001.0>] messages [] links [<5901.49.0>,<5901.4002.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4002.0> 2011-07-14 00:21:21 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4110.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4112.0> 2011-07-14 00:21:21 =============================================================================== ns_1@10.88.165.188:<5901.4112.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:21 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 175 times in the past 168.997407 secs (last seen 0.999412 secs ago PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4122.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4123.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4124.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4125.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4126.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4127.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4122.0> 2011-07-14 00:21:22 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4122.0> 2011-07-14 00:21:22 =============================================================================== ** Generic server <5901.4122.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.36027>}, init, {1310,617281,900515}, "default",#Port<5901.2909>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4122.0> 2011-07-14 00:21:22 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4122.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4001.0>] messages [] links [<5901.49.0>,<5901.4002.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4002.0> 2011-07-14 00:21:22 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4122.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:22 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 171 times in the past 169.656449 secs (last seen 1.000151 secs ago PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4131.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:22 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:22 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4132.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4133.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4134.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4135.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4136.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4131.0> 2011-07-14 00:21:22 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4131.0> 2011-07-14 00:21:22 =============================================================================== ** Generic server <5901.4131.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.36114>}, init, {1310,617282,403002}, "default",#Port<5901.2910>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4131.0> 2011-07-14 00:21:22 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4131.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4001.0>] messages [] links [<5901.49.0>,<5901.4002.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4002.0> 2011-07-14 00:21:22 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4131.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4133.0> 2011-07-14 00:21:22 =============================================================================== ns_1@10.88.165.188:<5901.4133.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:22 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 176 times in the past 169.997441 secs (last seen 1.000034 secs ago PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4140.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4141.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4142.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4143.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4144.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4145.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4140.0> 2011-07-14 00:21:23 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4140.0> 2011-07-14 00:21:23 =============================================================================== ** Generic server <5901.4140.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.36196>}, init, {1310,617282,902232}, "default",#Port<5901.2911>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4140.0> 2011-07-14 00:21:23 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4140.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4001.0>] messages [] links [<5901.49.0>,<5901.4002.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4002.0> 2011-07-14 00:21:23 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4140.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:23 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 172 times in the past 170.65638 secs (last seen 0.999931 secs ago PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4151.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4152.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4153.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:23 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:23 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4154.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4155.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4156.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4151.0> 2011-07-14 00:21:23 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4151.0> 2011-07-14 00:21:23 =============================================================================== ** Generic server <5901.4151.0> terminating ** Last message in was list_vbuckets_prevstate ** When Server state == {state,{interval,#Ref<5901.0.0.36334>}, init, {1310,617283,402894}, "default",#Port<5901.2913>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4151.0> 2011-07-14 00:21:23 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4151.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4001.0>] messages [] links [<5901.49.0>,<5901.4002.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6671 SUPERVISOR REPORT <5901.4002.0> 2011-07-14 00:21:23 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4151.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker CRASH REPORT <5901.4163.0> 2011-07-14 00:21:23 =============================================================================== Crashing process initial_call {ns_janitor,cleanup,['Argument__1','Argument__2']} pid <5901.4163.0> registered_name [] error_info {exit,{{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, {gen_server,call, [{'ns_memcached-default','ns_1@10.88.165.188'}, list_vbuckets_prevstate,30000]}}, [{gen_server,call,3}, {ns_janitor,'-read_existing_map/4-lc$^0/1-0-',2}, {ns_janitor,read_existing_map,4}, {ns_janitor,cleanup,2}, {proc_lib,init_p_do_apply,3}]} ancestors [<5901.304.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<5901.52.0>] messages [] links [<5901.304.0>] dictionary [] trap_exit false status running heap_size 10946 stack_size 24 reductions 224 INFO REPORT <5901.227.0> 2011-07-14 00:21:23 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 177 times in the past 170.996264 secs (last seen 0.998823 secs ago INFO REPORT <5901.304.0> 2011-07-14 00:21:23 =============================================================================== ns_1@10.88.165.188:<5901.304.0>:ns_orchestrator:178: Janitor run exited for bucket "default" with reason {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, [{'ns_memcached-default', 'ns_1@10.88.165.188'}, list_vbuckets_prevstate, 30000]}} ERROR REPORT <5901.4153.0> 2011-07-14 00:21:23 =============================================================================== ns_1@10.88.165.188:<5901.4153.0>:stats_collector:94: Exception in stats collector: {exit, {noproc, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4169.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4170.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4172.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4173.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4175.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4176.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4169.0> 2011-07-14 00:21:24 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4169.0> 2011-07-14 00:21:24 =============================================================================== ** Generic server <5901.4169.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.36448>}, init, {1310,617283,899597}, "default",#Port<5901.2915>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4169.0> 2011-07-14 00:21:24 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4169.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4001.0>] messages [] links [<5901.49.0>,<5901.4002.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.4002.0> 2011-07-14 00:21:24 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4169.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:24 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 173 times in the past 171.656375 secs (last seen 0.999995 secs ago PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4180.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4181.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4182.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4183.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4184.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4185.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4180.0> 2011-07-14 00:21:24 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4180.0> 2011-07-14 00:21:24 =============================================================================== ** Generic server <5901.4180.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.36547>}, init, {1310,617284,402892}, "default",#Port<5901.2916>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4180.0> 2011-07-14 00:21:24 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4180.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4001.0>] messages [] links [<5901.49.0>,<5901.4002.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4002.0> 2011-07-14 00:21:24 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4180.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4182.0> 2011-07-14 00:21:24 =============================================================================== ns_1@10.88.165.188:<5901.4182.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:24 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 178 times in the past 171.997471 secs (last seen 1.001207 secs ago PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4189.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:24 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:24 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4190.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4191.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4192.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4193.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4194.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4189.0> 2011-07-14 00:21:25 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4189.0> 2011-07-14 00:21:25 =============================================================================== ** Generic server <5901.4189.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.36623>}, init, {1310,617284,900490}, "default",#Port<5901.2917>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4189.0> 2011-07-14 00:21:25 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4189.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4001.0>] messages [] links [<5901.49.0>,<5901.4002.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4002.0> 2011-07-14 00:21:25 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4189.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:25 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 174 times in the past 172.656402 secs (last seen 1.000027 secs ago PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4198.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4199.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4200.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4201.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4202.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4203.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4198.0> 2011-07-14 00:21:25 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4198.0> 2011-07-14 00:21:25 =============================================================================== ** Generic server <5901.4198.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.36711>}, init, {1310,617285,403006}, "default",#Port<5901.2918>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4198.0> 2011-07-14 00:21:25 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4198.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4001.0>] messages [] links [<5901.49.0>,<5901.4002.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4002.0> 2011-07-14 00:21:25 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4198.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4200.0> 2011-07-14 00:21:25 =============================================================================== ns_1@10.88.165.188:<5901.4200.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:25 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 179 times in the past 172.997344 secs (last seen 0.999873 secs ago PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4210.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4211.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4212.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:25 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:25 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4213.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4214.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4215.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4210.0> 2011-07-14 00:21:26 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4210.0> 2011-07-14 00:21:26 =============================================================================== ** Generic server <5901.4210.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.36792>}, init, {1310,617285,900417}, "default",#Port<5901.2919>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4210.0> 2011-07-14 00:21:26 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4210.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4001.0>] messages [] links [<5901.49.0>,<5901.4002.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4002.0> 2011-07-14 00:21:26 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4210.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:26 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 175 times in the past 173.656423 secs (last seen 1.000021 secs ago PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4221.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4222.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4223.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4224.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4225.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4002.0> 2011-07-14 00:21:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4226.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4221.0> 2011-07-14 00:21:26 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4221.0> 2011-07-14 00:21:26 =============================================================================== ** Generic server <5901.4221.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.36942>}, init, {1310,617286,403022}, "default",#Port<5901.2921>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4221.0> 2011-07-14 00:21:26 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4221.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4001.0>] messages [] links [<5901.49.0>,<5901.4002.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4002.0> 2011-07-14 00:21:26 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4221.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker SUPERVISOR REPORT <5901.4002.0> 2011-07-14 00:21:26 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext shutdown reason reached_max_restart_intensity pid <5901.4221.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4223.0> 2011-07-14 00:21:26 =============================================================================== ns_1@10.88.165.188:<5901.4223.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:26 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 180 times in the past 173.997297 secs (last seen 0.999953 secs ago INFO REPORT <5901.4001.0> 2011-07-14 00:21:26 =============================================================================== ns_1@10.88.165.188:<5901.4001.0>:single_bucket_sup:24: per-bucket supervisor for "default" died with reason shutdown SUPERVISOR REPORT <5901.273.0> 2011-07-14 00:21:26 =============================================================================== Reporting supervisor {local,ns_bucket_sup} Child process errorContext child_terminated reason shutdown pid <5901.4001.0> name {per_bucket_sup,"default"} mfargs {single_bucket_sup,start_link,["default"]} restart_type permanent shutdown infinity child_type supervisor PROGRESS REPORT <5901.273.0> 2011-07-14 00:21:26 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<5901.4230.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4232.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4233.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4234.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4235.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4236.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4237.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4232.0> 2011-07-14 00:21:27 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4232.0> 2011-07-14 00:21:27 =============================================================================== ** Generic server <5901.4232.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.37025>}, init, {1310,617286,900796}, "default",#Port<5901.2922>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4232.0> 2011-07-14 00:21:27 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4232.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4230.0>] messages [] links [<5901.49.0>,<5901.4231.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4231.0> 2011-07-14 00:21:27 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4232.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:27 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 176 times in the past 174.656537 secs (last seen 1.000114 secs ago PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4241.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:27 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:27 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4242.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4243.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4244.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4245.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4246.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4241.0> 2011-07-14 00:21:27 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4241.0> 2011-07-14 00:21:27 =============================================================================== ** Generic server <5901.4241.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.37107>}, init, {1310,617287,403032}, "default",#Port<5901.2923>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4241.0> 2011-07-14 00:21:27 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4241.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4230.0>] messages [] links [<5901.49.0>,<5901.4231.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4231.0> 2011-07-14 00:21:27 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4241.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4243.0> 2011-07-14 00:21:27 =============================================================================== ns_1@10.88.165.188:<5901.4243.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:27 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 181 times in the past 174.997362 secs (last seen 1.000065 secs ago PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4253.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4254.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4255.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4256.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4257.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4258.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4253.0> 2011-07-14 00:21:28 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4253.0> 2011-07-14 00:21:28 =============================================================================== ** Generic server <5901.4253.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.37199>}, init, {1310,617287,900491}, "default",#Port<5901.2924>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4253.0> 2011-07-14 00:21:28 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4253.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4230.0>] messages [] links [<5901.49.0>,<5901.4231.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4231.0> 2011-07-14 00:21:28 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4253.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:28 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 177 times in the past 175.656563 secs (last seen 1.000026 secs ago PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4262.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:28 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:28 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4263.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4264.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4265.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4266.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4267.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4262.0> 2011-07-14 00:21:28 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4262.0> 2011-07-14 00:21:28 =============================================================================== ** Generic server <5901.4262.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.37285>}, init, {1310,617288,403127}, "default",#Port<5901.2925>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4262.0> 2011-07-14 00:21:28 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4262.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4230.0>] messages [] links [<5901.49.0>,<5901.4231.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6665 SUPERVISOR REPORT <5901.4231.0> 2011-07-14 00:21:28 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4262.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4264.0> 2011-07-14 00:21:28 =============================================================================== ns_1@10.88.165.188:<5901.4264.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:28 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 182 times in the past 175.997239 secs (last seen 0.999877 secs ago PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4275.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4277.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4278.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4279.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4280.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4281.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4275.0> 2011-07-14 00:21:29 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4275.0> 2011-07-14 00:21:29 =============================================================================== ** Generic server <5901.4275.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.37454>}, init, {1310,617288,900336}, "default",#Port<5901.2927>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4275.0> 2011-07-14 00:21:29 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4275.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4230.0>] messages [] links [<5901.49.0>,<5901.4231.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.4231.0> 2011-07-14 00:21:29 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4275.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:29 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 178 times in the past 176.6566 secs (last seen 1.000037 secs ago PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4288.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4289.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4290.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4291.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4292.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4293.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4288.0> 2011-07-14 00:21:29 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4288.0> 2011-07-14 00:21:29 =============================================================================== ** Generic server <5901.4288.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.37608>}, init, {1310,617289,403119}, "default",#Port<5901.2929>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4288.0> 2011-07-14 00:21:29 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4288.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4230.0>] messages [] links [<5901.49.0>,<5901.4231.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4231.0> 2011-07-14 00:21:29 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4288.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4290.0> 2011-07-14 00:21:29 =============================================================================== ns_1@10.88.165.188:<5901.4290.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:29 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 183 times in the past 176.997267 secs (last seen 1.000028 secs ago INFO REPORT <5901.269.0> 2011-07-14 00:21:29 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:29 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4300.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4301.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4302.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4303.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4304.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4305.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4300.0> 2011-07-14 00:21:30 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4300.0> 2011-07-14 00:21:30 =============================================================================== ** Generic server <5901.4300.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.37689>}, init, {1310,617289,900400}, "default",#Port<5901.2930>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4300.0> 2011-07-14 00:21:30 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4300.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4230.0>] messages [] links [<5901.49.0>,<5901.4231.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4231.0> 2011-07-14 00:21:30 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4300.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:30 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 179 times in the past 177.656875 secs (last seen 1.000275 secs ago PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4309.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4310.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4311.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4312.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4313.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4314.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4309.0> 2011-07-14 00:21:30 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4309.0> 2011-07-14 00:21:30 =============================================================================== ** Generic server <5901.4309.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.37775>}, init, {1310,617290,403464}, "default",#Port<5901.2931>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4309.0> 2011-07-14 00:21:30 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4309.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4230.0>] messages [] links [<5901.49.0>,<5901.4231.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4231.0> 2011-07-14 00:21:30 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4309.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4311.0> 2011-07-14 00:21:30 =============================================================================== ns_1@10.88.165.188:<5901.4311.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:30 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 184 times in the past 177.997375 secs (last seen 1.000108 secs ago PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4318.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4319.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4320.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4321.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4322.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4323.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4318.0> 2011-07-14 00:21:31 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4318.0> 2011-07-14 00:21:31 =============================================================================== ** Generic server <5901.4318.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.37857>}, init, {1310,617290,900939}, "default",#Port<5901.2932>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4318.0> 2011-07-14 00:21:31 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4318.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4230.0>] messages [] links [<5901.49.0>,<5901.4231.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4231.0> 2011-07-14 00:21:31 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4318.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:31 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 180 times in the past 178.660845 secs (last seen 1.00397 secs ago PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4327.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:31 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:31 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4328.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4329.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4330.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4331.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4332.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4327.0> 2011-07-14 00:21:31 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4327.0> 2011-07-14 00:21:31 =============================================================================== ** Generic server <5901.4327.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.37939>}, init, {1310,617291,407458}, "default",#Port<5901.2933>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4327.0> 2011-07-14 00:21:31 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4327.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4230.0>] messages [] links [<5901.49.0>,<5901.4231.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4231.0> 2011-07-14 00:21:31 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4327.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4329.0> 2011-07-14 00:21:31 =============================================================================== ns_1@10.88.165.188:<5901.4329.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:31 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 185 times in the past 178.997297 secs (last seen 0.999922 secs ago PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4339.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4340.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4341.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4342.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4343.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4344.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4339.0> 2011-07-14 00:21:32 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4339.0> 2011-07-14 00:21:32 =============================================================================== ** Generic server <5901.4339.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.38027>}, init, {1310,617291,900411}, "default",#Port<5901.2934>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4339.0> 2011-07-14 00:21:32 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4339.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4230.0>] messages [] links [<5901.49.0>,<5901.4231.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4231.0> 2011-07-14 00:21:32 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4339.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:32 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 181 times in the past 179.656853 secs (last seen 0.996008 secs ago PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4349.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4350.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4351.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4352.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4353.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4354.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:32 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:32 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.4349.0> 2011-07-14 00:21:32 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4349.0> 2011-07-14 00:21:32 =============================================================================== ** Generic server <5901.4349.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.38153>}, init, {1310,617292,403406}, "default",#Port<5901.2936>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4349.0> 2011-07-14 00:21:32 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4349.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4230.0>] messages [check_started] links [<5901.49.0>,<5901.4231.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4231.0> 2011-07-14 00:21:32 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4349.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4351.0> 2011-07-14 00:21:32 =============================================================================== ns_1@10.88.165.188:<5901.4351.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:32 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 186 times in the past 180.002026 secs (last seen 1.004729 secs ago PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4359.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4360.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4361.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4362.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4363.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4364.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4359.0> 2011-07-14 00:21:33 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4359.0> 2011-07-14 00:21:33 =============================================================================== ** Generic server <5901.4359.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.38249>}, init, {1310,617292,905115}, "default",#Port<5901.2937>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4359.0> 2011-07-14 00:21:33 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4359.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4230.0>] messages [] links [<5901.49.0>,<5901.4231.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4231.0> 2011-07-14 00:21:33 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4359.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:33 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 182 times in the past 180.660926 secs (last seen 1.004073 secs ago PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4368.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4369.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4370.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4371.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4372.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4373.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4368.0> 2011-07-14 00:21:33 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4368.0> 2011-07-14 00:21:33 =============================================================================== ** Generic server <5901.4368.0> terminating ** Last message in was list_vbuckets_prevstate ** When Server state == {state,{interval,#Ref<5901.0.0.38337>}, init, {1310,617293,407488}, "default",#Port<5901.2938>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4368.0> 2011-07-14 00:21:33 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4368.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4230.0>] messages [] links [<5901.49.0>,<5901.4231.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6671 SUPERVISOR REPORT <5901.4231.0> 2011-07-14 00:21:33 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4368.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker CRASH REPORT <5901.4380.0> 2011-07-14 00:21:33 =============================================================================== Crashing process initial_call {ns_janitor,cleanup,['Argument__1','Argument__2']} pid <5901.4380.0> registered_name [] error_info {exit,{{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, {gen_server,call, [{'ns_memcached-default','ns_1@10.88.165.188'}, list_vbuckets_prevstate,30000]}}, [{gen_server,call,3}, {ns_janitor,'-read_existing_map/4-lc$^0/1-0-',2}, {ns_janitor,read_existing_map,4}, {ns_janitor,cleanup,2}, {proc_lib,init_p_do_apply,3}]} ancestors [<5901.304.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<5901.52.0>] messages [] links [<5901.304.0>] dictionary [] trap_exit false status running heap_size 10946 stack_size 24 reductions 224 INFO REPORT <5901.227.0> 2011-07-14 00:21:33 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 187 times in the past 180.995763 secs (last seen 0.993737 secs ago INFO REPORT <5901.304.0> 2011-07-14 00:21:33 =============================================================================== ns_1@10.88.165.188:<5901.304.0>:ns_orchestrator:178: Janitor run exited for bucket "default" with reason {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, [{'ns_memcached-default', 'ns_1@10.88.165.188'}, list_vbuckets_prevstate, 30000]}} ERROR REPORT <5901.4370.0> 2011-07-14 00:21:33 =============================================================================== ns_1@10.88.165.188:<5901.4370.0>:stats_collector:94: Exception in stats collector: {exit, {noproc, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4386.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4387.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4389.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4390.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4392.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4393.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:33 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:33 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.4386.0> 2011-07-14 00:21:34 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4386.0> 2011-07-14 00:21:34 =============================================================================== ** Generic server <5901.4386.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.38447>}, init, {1310,617293,899626}, "default",#Port<5901.2940>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4386.0> 2011-07-14 00:21:34 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4386.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4230.0>] messages [] links [<5901.49.0>,<5901.4231.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.4231.0> 2011-07-14 00:21:34 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4386.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:34 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 183 times in the past 181.653704 secs (last seen 0.992778 secs ago PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4397.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4398.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4399.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4400.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4401.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4402.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4397.0> 2011-07-14 00:21:34 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4397.0> 2011-07-14 00:21:34 =============================================================================== ** Generic server <5901.4397.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.38544>}, init, {1310,617294,400366}, "default",#Port<5901.2941>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4397.0> 2011-07-14 00:21:34 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4397.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4230.0>] messages [] links [<5901.49.0>,<5901.4231.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4231.0> 2011-07-14 00:21:34 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4397.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4399.0> 2011-07-14 00:21:34 =============================================================================== ns_1@10.88.165.188:<5901.4399.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:34 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 188 times in the past 181.997387 secs (last seen 1.001624 secs ago PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4406.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4407.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4408.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4409.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4410.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4411.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4406.0> 2011-07-14 00:21:35 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4406.0> 2011-07-14 00:21:35 =============================================================================== ** Generic server <5901.4406.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.38626>}, init, {1310,617294,900379}, "default",#Port<5901.2942>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4406.0> 2011-07-14 00:21:35 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4406.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4230.0>] messages [] links [<5901.49.0>,<5901.4231.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4231.0> 2011-07-14 00:21:35 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4406.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:35 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 184 times in the past 182.657019 secs (last seen 1.003315 secs ago PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4416.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:35 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:35 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4417.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4418.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4419.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4420.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4421.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4416.0> 2011-07-14 00:21:35 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4416.0> 2011-07-14 00:21:35 =============================================================================== ** Generic server <5901.4416.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.38745>}, init, {1310,617295,403673}, "default",#Port<5901.2946>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4416.0> 2011-07-14 00:21:35 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4416.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4230.0>] messages [] links [<5901.49.0>,<5901.4231.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4231.0> 2011-07-14 00:21:35 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4416.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4418.0> 2011-07-14 00:21:35 =============================================================================== ns_1@10.88.165.188:<5901.4418.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:35 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 189 times in the past 182.999053 secs (last seen 1.001666 secs ago PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4429.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4430.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4431.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4432.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4433.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4434.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4429.0> 2011-07-14 00:21:36 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4429.0> 2011-07-14 00:21:36 =============================================================================== ** Generic server <5901.4429.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.38852>}, init, {1310,617295,902141}, "default",#Port<5901.2947>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4429.0> 2011-07-14 00:21:36 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4429.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4230.0>] messages [] links [<5901.49.0>,<5901.4231.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4231.0> 2011-07-14 00:21:36 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4429.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:36 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 185 times in the past 183.657107 secs (last seen 1.000088 secs ago PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4438.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4439.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4440.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:36 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:36 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4441.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4442.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4443.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4438.0> 2011-07-14 00:21:36 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4438.0> 2011-07-14 00:21:36 =============================================================================== ** Generic server <5901.4438.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.38939>}, init, {1310,617296,403645}, "default",#Port<5901.2948>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4438.0> 2011-07-14 00:21:36 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4438.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4230.0>] messages [] links [<5901.49.0>,<5901.4231.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4231.0> 2011-07-14 00:21:36 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4438.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4440.0> 2011-07-14 00:21:36 =============================================================================== ns_1@10.88.165.188:<5901.4440.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:36 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 190 times in the past 183.997463 secs (last seen 0.99841 secs ago PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4447.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4448.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4449.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4450.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4451.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4231.0> 2011-07-14 00:21:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4452.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4447.0> 2011-07-14 00:21:37 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4447.0> 2011-07-14 00:21:37 =============================================================================== ** Generic server <5901.4447.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.39015>}, init, {1310,617296,900652}, "default",#Port<5901.2949>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4447.0> 2011-07-14 00:21:37 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4447.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4230.0>] messages [] links [<5901.49.0>,<5901.4231.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4231.0> 2011-07-14 00:21:37 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4447.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker SUPERVISOR REPORT <5901.4231.0> 2011-07-14 00:21:37 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext shutdown reason reached_max_restart_intensity pid <5901.4447.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:37 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 186 times in the past 184.657116 secs (last seen 1.000009 secs ago INFO REPORT <5901.4230.0> 2011-07-14 00:21:37 =============================================================================== ns_1@10.88.165.188:<5901.4230.0>:single_bucket_sup:24: per-bucket supervisor for "default" died with reason shutdown SUPERVISOR REPORT <5901.273.0> 2011-07-14 00:21:37 =============================================================================== Reporting supervisor {local,ns_bucket_sup} Child process errorContext child_terminated reason shutdown pid <5901.4230.0> name {per_bucket_sup,"default"} mfargs {single_bucket_sup,start_link,["default"]} restart_type permanent shutdown infinity child_type supervisor PROGRESS REPORT <5901.273.0> 2011-07-14 00:21:37 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<5901.4456.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4458.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4459.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4460.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4461.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4462.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4463.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4458.0> 2011-07-14 00:21:37 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4458.0> 2011-07-14 00:21:37 =============================================================================== ** Generic server <5901.4458.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.39103>}, init, {1310,617297,403920}, "default",#Port<5901.2950>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4458.0> 2011-07-14 00:21:37 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4458.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4456.0>] messages [] links [<5901.49.0>,<5901.4457.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4457.0> 2011-07-14 00:21:37 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4458.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4460.0> 2011-07-14 00:21:37 =============================================================================== ns_1@10.88.165.188:<5901.4460.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:37 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 191 times in the past 184.997341 secs (last seen 0.999878 secs ago PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4470.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4471.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4472.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:37 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:37 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4473.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4474.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4475.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4470.0> 2011-07-14 00:21:38 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4470.0> 2011-07-14 00:21:38 =============================================================================== ** Generic server <5901.4470.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.39184>}, init, {1310,617297,900385}, "default",#Port<5901.2951>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4470.0> 2011-07-14 00:21:38 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4470.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4456.0>] messages [] links [<5901.49.0>,<5901.4457.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4457.0> 2011-07-14 00:21:38 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4470.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:38 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 187 times in the past 185.654468 secs (last seen 0.997352 secs ago PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4480.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4481.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4482.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4483.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4484.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4485.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4480.0> 2011-07-14 00:21:38 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4480.0> 2011-07-14 00:21:38 =============================================================================== ** Generic server <5901.4480.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.39315>}, init, {1310,617298,401074}, "default",#Port<5901.2953>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4480.0> 2011-07-14 00:21:38 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4480.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4456.0>] messages [] links [<5901.49.0>,<5901.4457.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6665 SUPERVISOR REPORT <5901.4457.0> 2011-07-14 00:21:38 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4480.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4482.0> 2011-07-14 00:21:38 =============================================================================== ns_1@10.88.165.188:<5901.4482.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:38 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 192 times in the past 185.997311 secs (last seen 0.99997 secs ago PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4493.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4494.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4496.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4497.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4498.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4499.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4493.0> 2011-07-14 00:21:39 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4493.0> 2011-07-14 00:21:39 =============================================================================== ** Generic server <5901.4493.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.39433>}, init, {1310,617298,900264}, "default",#Port<5901.2954>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4493.0> 2011-07-14 00:21:39 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4493.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4456.0>] messages [] links [<5901.49.0>,<5901.4457.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.4457.0> 2011-07-14 00:21:39 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4493.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:39 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 188 times in the past 186.654496 secs (last seen 1.000028 secs ago PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4504.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:39 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:39 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4505.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4506.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4507.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4508.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4509.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4504.0> 2011-07-14 00:21:39 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4504.0> 2011-07-14 00:21:39 =============================================================================== ** Generic server <5901.4504.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.39525>}, init, {1310,617299,401062}, "default",#Port<5901.2956>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4504.0> 2011-07-14 00:21:39 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4504.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4456.0>] messages [] links [<5901.49.0>,<5901.4457.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4457.0> 2011-07-14 00:21:39 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4504.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4506.0> 2011-07-14 00:21:39 =============================================================================== ns_1@10.88.165.188:<5901.4506.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:39 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 193 times in the past 186.99742 secs (last seen 1.000109 secs ago PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4516.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4517.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4518.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4519.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4520.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4521.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4516.0> 2011-07-14 00:21:40 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4516.0> 2011-07-14 00:21:40 =============================================================================== ** Generic server <5901.4516.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.39612>}, init, {1310,617299,900470}, "default",#Port<5901.2957>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4516.0> 2011-07-14 00:21:40 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4516.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4456.0>] messages [] links [<5901.49.0>,<5901.4457.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4457.0> 2011-07-14 00:21:40 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4516.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:40 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 189 times in the past 187.654578 secs (last seen 1.000082 secs ago PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4525.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:40 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:40 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4526.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4527.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4528.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4529.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4530.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4525.0> 2011-07-14 00:21:40 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4525.0> 2011-07-14 00:21:40 =============================================================================== ** Generic server <5901.4525.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.39699>}, init, {1310,617300,401068}, "default",#Port<5901.2958>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4525.0> 2011-07-14 00:21:40 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4525.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4456.0>] messages [] links [<5901.49.0>,<5901.4457.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4457.0> 2011-07-14 00:21:40 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4525.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4527.0> 2011-07-14 00:21:40 =============================================================================== ns_1@10.88.165.188:<5901.4527.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:40 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 194 times in the past 187.99767 secs (last seen 1.00025 secs ago PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4534.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4535.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4536.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4537.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4538.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4539.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4534.0> 2011-07-14 00:21:41 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4534.0> 2011-07-14 00:21:41 =============================================================================== ** Generic server <5901.4534.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.39775>}, init, {1310,617300,900881}, "default",#Port<5901.2959>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4534.0> 2011-07-14 00:21:41 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4534.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4456.0>] messages [] links [<5901.49.0>,<5901.4457.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4457.0> 2011-07-14 00:21:41 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4534.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:41 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 190 times in the past 188.654904 secs (last seen 1.000326 secs ago PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4544.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4545.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4546.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4547.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4548.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4549.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4544.0> 2011-07-14 00:21:41 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4544.0> 2011-07-14 00:21:41 =============================================================================== ** Generic server <5901.4544.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.39899>}, init, {1310,617301,401469}, "default",#Port<5901.2961>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4544.0> 2011-07-14 00:21:41 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4544.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4456.0>] messages [] links [<5901.49.0>,<5901.4457.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4457.0> 2011-07-14 00:21:41 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4544.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4546.0> 2011-07-14 00:21:41 =============================================================================== ns_1@10.88.165.188:<5901.4546.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:41 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 195 times in the past 188.99755 secs (last seen 0.99988 secs ago PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4556.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4557.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4558.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:41 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:41 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4559.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4560.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4561.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4556.0> 2011-07-14 00:21:42 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4556.0> 2011-07-14 00:21:42 =============================================================================== ** Generic server <5901.4556.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.39980>}, init, {1310,617301,900625}, "default",#Port<5901.2962>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4556.0> 2011-07-14 00:21:42 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4556.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4456.0>] messages [] links [<5901.49.0>,<5901.4457.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4457.0> 2011-07-14 00:21:42 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4556.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:42 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 191 times in the past 189.654642 secs (last seen 0.999738 secs ago PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4566.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4567.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4568.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4569.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4570.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4571.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4566.0> 2011-07-14 00:21:42 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4566.0> 2011-07-14 00:21:42 =============================================================================== ** Generic server <5901.4566.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.40086>}, init, {1310,617302,401168}, "default",#Port<5901.2963>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4566.0> 2011-07-14 00:21:42 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4566.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4456.0>] messages [] links [<5901.49.0>,<5901.4457.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4457.0> 2011-07-14 00:21:42 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4566.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4568.0> 2011-07-14 00:21:42 =============================================================================== ns_1@10.88.165.188:<5901.4568.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:42 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 196 times in the past 189.997686 secs (last seen 1.000136 secs ago PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4575.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4576.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4577.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4578.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4579.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4580.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4575.0> 2011-07-14 00:21:43 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4575.0> 2011-07-14 00:21:43 =============================================================================== ** Generic server <5901.4575.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.40169>}, init, {1310,617302,901147}, "default",#Port<5901.2964>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4575.0> 2011-07-14 00:21:43 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4575.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4456.0>] messages [] links [<5901.49.0>,<5901.4457.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4457.0> 2011-07-14 00:21:43 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4575.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:43 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 192 times in the past 190.657596 secs (last seen 1.002954 secs ago PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4584.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:43 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:43 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4585.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4586.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4587.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4588.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4589.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4584.0> 2011-07-14 00:21:43 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4584.0> 2011-07-14 00:21:43 =============================================================================== ** Generic server <5901.4584.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.40251>}, init, {1310,617303,404156}, "default",#Port<5901.2965>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4584.0> 2011-07-14 00:21:43 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4584.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4456.0>] messages [] links [<5901.49.0>,<5901.4457.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6671 SUPERVISOR REPORT <5901.4457.0> 2011-07-14 00:21:43 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4584.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4586.0> 2011-07-14 00:21:43 =============================================================================== ns_1@10.88.165.188:<5901.4586.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:43 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 197 times in the past 190.997237 secs (last seen 0.999551 secs ago PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4603.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4604.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4606.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4603.0> 2011-07-14 00:21:43 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4603.0> 2011-07-14 00:21:43 =============================================================================== ** Generic server <5901.4603.0> terminating ** Last message in was list_vbuckets_prevstate ** When Server state == {state,{interval,#Ref<5901.0.0.40444>}, init, {1310,617303,900178}, "default",#Port<5901.2968>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4603.0> 2011-07-14 00:21:43 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4603.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4456.0>] messages [] links [<5901.49.0>,<5901.4457.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6709 CRASH REPORT <5901.4596.0> 2011-07-14 00:21:43 =============================================================================== Crashing process initial_call {ns_janitor,cleanup,['Argument__1','Argument__2']} pid <5901.4596.0> registered_name [] error_info {exit,{{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, {gen_server,call, [{'ns_memcached-default','ns_1@10.88.165.188'}, list_vbuckets_prevstate,30000]}}, [{gen_server,call,3}, {ns_janitor,'-read_existing_map/4-lc$^0/1-0-',2}, {ns_janitor,read_existing_map,4}, {ns_janitor,cleanup,2}, {proc_lib,init_p_do_apply,3}]} ancestors [<5901.304.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<5901.52.0>] messages [] links [<5901.304.0>] dictionary [] trap_exit false status running heap_size 10946 stack_size 24 reductions 224 INFO REPORT <5901.227.0> 2011-07-14 00:21:43 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 198 times in the past 191.006977 secs (last seen 0.00974 secs ago INFO REPORT <5901.304.0> 2011-07-14 00:21:43 =============================================================================== ns_1@10.88.165.188:<5901.304.0>:ns_orchestrator:178: Janitor run exited for bucket "default" with reason {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, [{'ns_memcached-default', 'ns_1@10.88.165.188'}, list_vbuckets_prevstate, 30000]}} PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4607.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4609.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4610.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] SUPERVISOR REPORT <5901.4457.0> 2011-07-14 00:21:43 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4603.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4611.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4612.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4613.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4614.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4615.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4616.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:44 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:44 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.4611.0> 2011-07-14 00:21:44 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4611.0> 2011-07-14 00:21:44 =============================================================================== ** Generic server <5901.4611.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.40515>}, init, {1310,617303,909496}, "default",#Port<5901.2969>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4611.0> 2011-07-14 00:21:44 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4611.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4456.0>] messages [] links [<5901.49.0>,<5901.4457.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4457.0> 2011-07-14 00:21:44 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4611.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:44 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 193 times in the past 191.665841 secs (last seen 1.008245 secs ago PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4621.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4622.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4623.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4624.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4625.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4626.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4621.0> 2011-07-14 00:21:44 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4621.0> 2011-07-14 00:21:44 =============================================================================== ** Generic server <5901.4621.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.40635>}, init, {1310,617304,412436}, "default",#Port<5901.2971>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4621.0> 2011-07-14 00:21:44 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4621.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4456.0>] messages [] links [<5901.49.0>,<5901.4457.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4457.0> 2011-07-14 00:21:44 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4621.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4623.0> 2011-07-14 00:21:44 =============================================================================== ns_1@10.88.165.188:<5901.4623.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:44 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 199 times in the past 191.997779 secs (last seen 0.990802 secs ago PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4630.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4631.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4632.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4633.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4634.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4635.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4630.0> 2011-07-14 00:21:45 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4630.0> 2011-07-14 00:21:45 =============================================================================== ** Generic server <5901.4630.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.40713>}, init, {1310,617304,900752}, "default",#Port<5901.2972>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4630.0> 2011-07-14 00:21:45 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4630.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4456.0>] messages [] links [<5901.49.0>,<5901.4457.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4457.0> 2011-07-14 00:21:45 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4630.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:45 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 194 times in the past 192.657759 secs (last seen 0.991918 secs ago PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4640.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4641.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4642.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4643.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4644.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4645.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:45 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:45 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.4640.0> 2011-07-14 00:21:45 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4640.0> 2011-07-14 00:21:45 =============================================================================== ** Generic server <5901.4640.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.40820>}, init, {1310,617305,404351}, "default",#Port<5901.2973>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4640.0> 2011-07-14 00:21:45 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4640.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4456.0>] messages [] links [<5901.49.0>,<5901.4457.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4457.0> 2011-07-14 00:21:45 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4640.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4642.0> 2011-07-14 00:21:45 =============================================================================== ns_1@10.88.165.188:<5901.4642.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:45 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 200 times in the past 192.99783 secs (last seen 1.000051 secs ago PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4652.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4653.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4654.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4655.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4656.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4657.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4652.0> 2011-07-14 00:21:46 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4652.0> 2011-07-14 00:21:46 =============================================================================== ** Generic server <5901.4652.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.40901>}, init, {1310,617305,901022}, "default",#Port<5901.2974>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4652.0> 2011-07-14 00:21:46 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4652.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4456.0>] messages [] links [<5901.49.0>,<5901.4457.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4457.0> 2011-07-14 00:21:46 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4652.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:46 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 195 times in the past 193.657762 secs (last seen 1.000003 secs ago PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4661.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4662.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4663.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4664.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4665.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4666.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4661.0> 2011-07-14 00:21:46 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4661.0> 2011-07-14 00:21:46 =============================================================================== ** Generic server <5901.4661.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.40989>}, init, {1310,617306,404365}, "default",#Port<5901.2975>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4661.0> 2011-07-14 00:21:46 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4661.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4456.0>] messages [] links [<5901.49.0>,<5901.4457.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4457.0> 2011-07-14 00:21:46 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4661.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4663.0> 2011-07-14 00:21:46 =============================================================================== ns_1@10.88.165.188:<5901.4663.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:46 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 201 times in the past 193.997968 secs (last seen 1.000138 secs ago PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4670.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:46 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:46 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4671.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4672.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4673.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4674.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4457.0> 2011-07-14 00:21:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4675.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4670.0> 2011-07-14 00:21:47 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4670.0> 2011-07-14 00:21:47 =============================================================================== ** Generic server <5901.4670.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.41069>}, init, {1310,617306,901909}, "default",#Port<5901.2976>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4670.0> 2011-07-14 00:21:47 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4670.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4456.0>] messages [] links [<5901.49.0>,<5901.4457.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4457.0> 2011-07-14 00:21:47 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4670.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker SUPERVISOR REPORT <5901.4457.0> 2011-07-14 00:21:47 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext shutdown reason reached_max_restart_intensity pid <5901.4670.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:47 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 196 times in the past 194.65793 secs (last seen 1.000168 secs ago INFO REPORT <5901.4456.0> 2011-07-14 00:21:47 =============================================================================== ns_1@10.88.165.188:<5901.4456.0>:single_bucket_sup:24: per-bucket supervisor for "default" died with reason shutdown SUPERVISOR REPORT <5901.273.0> 2011-07-14 00:21:47 =============================================================================== Reporting supervisor {local,ns_bucket_sup} Child process errorContext child_terminated reason shutdown pid <5901.4456.0> name {per_bucket_sup,"default"} mfargs {single_bucket_sup,start_link,["default"]} restart_type permanent shutdown infinity child_type supervisor PROGRESS REPORT <5901.273.0> 2011-07-14 00:21:47 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<5901.4680.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4682.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4683.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4684.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4685.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4686.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4687.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4682.0> 2011-07-14 00:21:47 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4682.0> 2011-07-14 00:21:47 =============================================================================== ** Generic server <5901.4682.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.41190>}, init, {1310,617307,404874}, "default",#Port<5901.2978>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4682.0> 2011-07-14 00:21:47 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4682.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4680.0>] messages [] links [<5901.49.0>,<5901.4681.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4681.0> 2011-07-14 00:21:47 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4682.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4684.0> 2011-07-14 00:21:47 =============================================================================== ns_1@10.88.165.188:<5901.4684.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:47 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 202 times in the past 194.997958 secs (last seen 0.99999 secs ago PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4694.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4695.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4696.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:47 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:47 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4697.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4698.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4699.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4694.0> 2011-07-14 00:21:48 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4694.0> 2011-07-14 00:21:48 =============================================================================== ** Generic server <5901.4694.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.41275>}, init, {1310,617307,901048}, "default",#Port<5901.2979>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4694.0> 2011-07-14 00:21:48 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4694.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4680.0>] messages [] links [<5901.49.0>,<5901.4681.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4681.0> 2011-07-14 00:21:48 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4694.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:48 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 197 times in the past 195.657838 secs (last seen 0.999908 secs ago PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4704.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4705.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4706.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4707.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4708.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4709.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4704.0> 2011-07-14 00:21:48 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4704.0> 2011-07-14 00:21:48 =============================================================================== ** Generic server <5901.4704.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.41382>}, init, {1310,617308,404440}, "default",#Port<5901.2982>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4704.0> 2011-07-14 00:21:48 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4704.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4680.0>] messages [] links [<5901.49.0>,<5901.4681.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6665 SUPERVISOR REPORT <5901.4681.0> 2011-07-14 00:21:48 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4704.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4706.0> 2011-07-14 00:21:48 =============================================================================== ns_1@10.88.165.188:<5901.4706.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:48 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 203 times in the past 195.997969 secs (last seen 1.000011 secs ago PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4716.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4717.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4719.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4720.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4721.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4722.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4716.0> 2011-07-14 00:21:49 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4716.0> 2011-07-14 00:21:49 =============================================================================== ** Generic server <5901.4716.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.41476>}, init, {1310,617308,901952}, "default",#Port<5901.2983>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4716.0> 2011-07-14 00:21:49 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4716.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4680.0>] messages [] links [<5901.49.0>,<5901.4681.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.4681.0> 2011-07-14 00:21:49 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4716.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:49 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 198 times in the past 196.65798 secs (last seen 1.000142 secs ago PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4727.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4728.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4729.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:49 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:49 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4730.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4731.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4732.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4727.0> 2011-07-14 00:21:49 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4727.0> 2011-07-14 00:21:49 =============================================================================== ** Generic server <5901.4727.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.41572>}, init, {1310,617309,404584}, "default",#Port<5901.2984>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4727.0> 2011-07-14 00:21:49 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4727.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4680.0>] messages [] links [<5901.49.0>,<5901.4681.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4681.0> 2011-07-14 00:21:49 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4727.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4729.0> 2011-07-14 00:21:49 =============================================================================== ns_1@10.88.165.188:<5901.4729.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:49 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 204 times in the past 196.998138 secs (last seen 1.000169 secs ago PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4739.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4740.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4741.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4742.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4743.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4744.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4739.0> 2011-07-14 00:21:50 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4739.0> 2011-07-14 00:21:50 =============================================================================== ** Generic server <5901.4739.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.41653>}, init, {1310,617309,901304}, "default",#Port<5901.2985>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4739.0> 2011-07-14 00:21:50 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4739.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4680.0>] messages [] links [<5901.49.0>,<5901.4681.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4681.0> 2011-07-14 00:21:50 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4739.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:50 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 199 times in the past 197.65538 secs (last seen 0.9974 secs ago PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4749.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4750.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4751.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4752.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4753.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4754.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4749.0> 2011-07-14 00:21:50 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4749.0> 2011-07-14 00:21:50 =============================================================================== ** Generic server <5901.4749.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.41785>}, init, {1310,617310,401966}, "default",#Port<5901.2987>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4749.0> 2011-07-14 00:21:50 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4749.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4680.0>] messages [] links [<5901.49.0>,<5901.4681.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4681.0> 2011-07-14 00:21:50 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4749.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4751.0> 2011-07-14 00:21:50 =============================================================================== ns_1@10.88.165.188:<5901.4751.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:50 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 205 times in the past 197.998224 secs (last seen 1.000086 secs ago PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4758.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:50 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:50 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4759.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4760.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4761.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4762.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4763.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4758.0> 2011-07-14 00:21:51 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4758.0> 2011-07-14 00:21:51 =============================================================================== ** Generic server <5901.4758.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.41865>}, init, {1310,617310,901251}, "default",#Port<5901.2988>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4758.0> 2011-07-14 00:21:51 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4758.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4680.0>] messages [] links [<5901.49.0>,<5901.4681.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4681.0> 2011-07-14 00:21:51 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4758.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:51 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 200 times in the past 198.655249 secs (last seen 0.999869 secs ago PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4768.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4769.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4770.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4771.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4772.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4773.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4768.0> 2011-07-14 00:21:51 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4768.0> 2011-07-14 00:21:51 =============================================================================== ** Generic server <5901.4768.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.41969>}, init, {1310,617311,401811}, "default",#Port<5901.2989>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4768.0> 2011-07-14 00:21:51 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4768.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4680.0>] messages [] links [<5901.49.0>,<5901.4681.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4681.0> 2011-07-14 00:21:51 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4768.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4770.0> 2011-07-14 00:21:51 =============================================================================== ns_1@10.88.165.188:<5901.4770.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:51 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 206 times in the past 198.998805 secs (last seen 1.000581 secs ago INFO REPORT <5901.269.0> 2011-07-14 00:21:51 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:51 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4780.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4781.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4782.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4783.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4784.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4785.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4780.0> 2011-07-14 00:21:52 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4780.0> 2011-07-14 00:21:52 =============================================================================== ** Generic server <5901.4780.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.42055>}, init, {1310,617311,902019}, "default",#Port<5901.2990>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4780.0> 2011-07-14 00:21:52 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4780.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4680.0>] messages [] links [<5901.49.0>,<5901.4681.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4681.0> 2011-07-14 00:21:52 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4780.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:52 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 201 times in the past 199.658196 secs (last seen 1.002947 secs ago PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4789.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4790.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4791.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4792.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4793.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4794.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4789.0> 2011-07-14 00:21:52 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4789.0> 2011-07-14 00:21:52 =============================================================================== ** Generic server <5901.4789.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.42142>}, init, {1310,617312,404824}, "default",#Port<5901.2992>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4789.0> 2011-07-14 00:21:52 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4789.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4680.0>] messages [] links [<5901.49.0>,<5901.4681.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4681.0> 2011-07-14 00:21:52 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4789.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4791.0> 2011-07-14 00:21:52 =============================================================================== ns_1@10.88.165.188:<5901.4791.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:52 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 207 times in the past 199.998386 secs (last seen 0.999581 secs ago PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4798.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4799.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4800.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4801.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4802.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4803.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4798.0> 2011-07-14 00:21:53 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4798.0> 2011-07-14 00:21:53 =============================================================================== ** Generic server <5901.4798.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.42225>}, init, {1310,617312,901503}, "default",#Port<5901.2993>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4798.0> 2011-07-14 00:21:53 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4798.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4680.0>] messages [] links [<5901.49.0>,<5901.4681.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4681.0> 2011-07-14 00:21:53 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4798.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:53 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 202 times in the past 200.658262 secs (last seen 1.000066 secs ago PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4808.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4809.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:53 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:53 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4810.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4811.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4812.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4813.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4808.0> 2011-07-14 00:21:53 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4808.0> 2011-07-14 00:21:53 =============================================================================== ** Generic server <5901.4808.0> terminating ** Last message in was list_vbuckets_prevstate ** When Server state == {state,{interval,#Ref<5901.0.0.42347>}, init, {1310,617313,404878}, "default",#Port<5901.2995>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4808.0> 2011-07-14 00:21:53 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4808.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4680.0>] messages [{'$gen_call',{<5901.4810.0>,#Ref<5901.0.0.42448>},{stats,<<>>}}] links [<5901.49.0>,<5901.4681.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6671 SUPERVISOR REPORT <5901.4681.0> 2011-07-14 00:21:53 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4808.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker CRASH REPORT <5901.4820.0> 2011-07-14 00:21:53 =============================================================================== Crashing process initial_call {ns_janitor,cleanup,['Argument__1','Argument__2']} pid <5901.4820.0> registered_name [] error_info {exit,{{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, {gen_server,call, [{'ns_memcached-default','ns_1@10.88.165.188'}, list_vbuckets_prevstate,30000]}}, [{gen_server,call,3}, {ns_janitor,'-read_existing_map/4-lc$^0/1-0-',2}, {ns_janitor,read_existing_map,4}, {ns_janitor,cleanup,2}, {proc_lib,init_p_do_apply,3}]} ancestors [<5901.304.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<5901.52.0>] messages [] links [<5901.304.0>] dictionary [] trap_exit false status running heap_size 10946 stack_size 24 reductions 224 ERROR REPORT <5901.4810.0> 2011-07-14 00:21:53 =============================================================================== ns_1@10.88.165.188:<5901.4810.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:53 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 208 times in the past 200.99719 secs (last seen 0.998804 secs ago INFO REPORT <5901.304.0> 2011-07-14 00:21:53 =============================================================================== ns_1@10.88.165.188:<5901.304.0>:ns_orchestrator:178: Janitor run exited for bucket "default" with reason {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, [{'ns_memcached-default', 'ns_1@10.88.165.188'}, list_vbuckets_prevstate, 30000]}} PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4826.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4827.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4828.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4830.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4831.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4832.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4826.0> 2011-07-14 00:21:54 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4826.0> 2011-07-14 00:21:54 =============================================================================== ** Generic server <5901.4826.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.42458>}, init, {1310,617313,899953}, "default",#Port<5901.2997>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4826.0> 2011-07-14 00:21:54 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4826.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4680.0>] messages [] links [<5901.49.0>,<5901.4681.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.4681.0> 2011-07-14 00:21:54 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4826.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:54 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 203 times in the past 201.654336 secs (last seen 0.996074 secs ago PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4837.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4838.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4839.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4840.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4841.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4842.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4837.0> 2011-07-14 00:21:54 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4837.0> 2011-07-14 00:21:54 =============================================================================== ** Generic server <5901.4837.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.42557>}, init, {1310,617314,400933}, "default",#Port<5901.2998>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4837.0> 2011-07-14 00:21:54 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4837.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4680.0>] messages [] links [<5901.49.0>,<5901.4681.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4681.0> 2011-07-14 00:21:54 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4837.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4839.0> 2011-07-14 00:21:54 =============================================================================== ns_1@10.88.165.188:<5901.4839.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:54 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 209 times in the past 201.998546 secs (last seen 1.001356 secs ago PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4847.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:54 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:54 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4848.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4849.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4850.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4851.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4852.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4847.0> 2011-07-14 00:21:55 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4847.0> 2011-07-14 00:21:55 =============================================================================== ** Generic server <5901.4847.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.42657>}, init, {1310,617314,901560}, "default",#Port<5901.2999>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4847.0> 2011-07-14 00:21:55 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4847.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4680.0>] messages [] links [<5901.49.0>,<5901.4681.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4681.0> 2011-07-14 00:21:55 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4847.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:55 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 204 times in the past 202.655717 secs (last seen 1.001381 secs ago PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4856.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4857.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4858.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4859.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4860.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4861.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4856.0> 2011-07-14 00:21:55 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4856.0> 2011-07-14 00:21:55 =============================================================================== ** Generic server <5901.4856.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.42741>}, init, {1310,617315,402387}, "default",#Port<5901.3000>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4856.0> 2011-07-14 00:21:55 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4856.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4680.0>] messages [] links [<5901.49.0>,<5901.4681.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4681.0> 2011-07-14 00:21:55 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4856.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4858.0> 2011-07-14 00:21:55 =============================================================================== ns_1@10.88.165.188:<5901.4858.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:55 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 210 times in the past 202.999025 secs (last seen 1.000479 secs ago INFO REPORT <5901.269.0> 2011-07-14 00:21:55 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:55 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4868.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4869.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4870.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4871.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4872.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4873.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4868.0> 2011-07-14 00:21:56 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4868.0> 2011-07-14 00:21:56 =============================================================================== ** Generic server <5901.4868.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.42826>}, init, {1310,617315,902221}, "default",#Port<5901.3001>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4868.0> 2011-07-14 00:21:56 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4868.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4680.0>] messages [] links [<5901.49.0>,<5901.4681.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4681.0> 2011-07-14 00:21:56 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4868.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:56 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 205 times in the past 203.658409 secs (last seen 1.002692 secs ago PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4878.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4879.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4880.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4881.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4882.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4883.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4878.0> 2011-07-14 00:21:56 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4878.0> 2011-07-14 00:21:56 =============================================================================== ** Generic server <5901.4878.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.42953>}, init, {1310,617316,404978}, "default",#Port<5901.3003>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4878.0> 2011-07-14 00:21:56 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4878.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4680.0>] messages [] links [<5901.49.0>,<5901.4681.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4681.0> 2011-07-14 00:21:56 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4878.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4880.0> 2011-07-14 00:21:56 =============================================================================== ns_1@10.88.165.188:<5901.4880.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:56 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 211 times in the past 203.998593 secs (last seen 0.999568 secs ago PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4887.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4888.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4889.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4890.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4891.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4892.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4887.0> 2011-07-14 00:21:57 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4887.0> 2011-07-14 00:21:57 =============================================================================== ** Generic server <5901.4887.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.43031>}, init, {1310,617316,901715}, "default",#Port<5901.3004>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4887.0> 2011-07-14 00:21:57 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4887.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4680.0>] messages [] links [<5901.49.0>,<5901.4681.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4681.0> 2011-07-14 00:21:57 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4887.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:57 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 206 times in the past 204.655803 secs (last seen 0.997394 secs ago PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4896.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4897.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4898.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4899.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4900.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4681.0> 2011-07-14 00:21:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4901.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:57 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:57 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.4896.0> 2011-07-14 00:21:57 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4896.0> 2011-07-14 00:21:57 =============================================================================== ** Generic server <5901.4896.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.43117>}, init, {1310,617317,402378}, "default",#Port<5901.3005>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4896.0> 2011-07-14 00:21:57 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4896.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4680.0>] messages [] links [<5901.49.0>,<5901.4681.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4681.0> 2011-07-14 00:21:57 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4896.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker SUPERVISOR REPORT <5901.4681.0> 2011-07-14 00:21:57 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext shutdown reason reached_max_restart_intensity pid <5901.4896.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4898.0> 2011-07-14 00:21:57 =============================================================================== ns_1@10.88.165.188:<5901.4898.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:57 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 212 times in the past 204.998558 secs (last seen 0.999965 secs ago INFO REPORT <5901.4680.0> 2011-07-14 00:21:57 =============================================================================== ns_1@10.88.165.188:<5901.4680.0>:single_bucket_sup:24: per-bucket supervisor for "default" died with reason shutdown SUPERVISOR REPORT <5901.273.0> 2011-07-14 00:21:57 =============================================================================== Reporting supervisor {local,ns_bucket_sup} Child process errorContext child_terminated reason shutdown pid <5901.4680.0> name {per_bucket_sup,"default"} mfargs {single_bucket_sup,start_link,["default"]} restart_type permanent shutdown infinity child_type supervisor PROGRESS REPORT <5901.273.0> 2011-07-14 00:21:57 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<5901.4909.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:21:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4911.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:21:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4912.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:21:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4913.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:21:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4914.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:21:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4915.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:21:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4916.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.239.0> 2011-07-14 00:21:57 =============================================================================== ns_1@10.88.165.188:<5901.239.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.4911.0> 2011-07-14 00:21:58 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4911.0> 2011-07-14 00:21:58 =============================================================================== ** Generic server <5901.4911.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.43218>}, init, {1310,617317,902716}, "default",#Port<5901.3006>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4911.0> 2011-07-14 00:21:58 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4911.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4909.0>] messages [] links [<5901.49.0>,<5901.4910.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4910.0> 2011-07-14 00:21:58 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4911.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:58 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 207 times in the past 205.655786 secs (last seen 0.999983 secs ago PROGRESS REPORT <5901.4910.0> 2011-07-14 00:21:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4922.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:21:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4923.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:21:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4924.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:21:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4925.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:21:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4926.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:21:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4927.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4922.0> 2011-07-14 00:21:58 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4922.0> 2011-07-14 00:21:58 =============================================================================== ** Generic server <5901.4922.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.43316>}, init, {1310,617318,402359}, "default",#Port<5901.3008>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4922.0> 2011-07-14 00:21:58 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4922.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4909.0>] messages [] links [<5901.49.0>,<5901.4910.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6665 SUPERVISOR REPORT <5901.4910.0> 2011-07-14 00:21:58 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4922.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4924.0> 2011-07-14 00:21:58 =============================================================================== ns_1@10.88.165.188:<5901.4924.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:58 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 213 times in the past 205.997339 secs (last seen 0.998781 secs ago PROGRESS REPORT <5901.4910.0> 2011-07-14 00:21:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4935.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:21:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4936.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:21:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4938.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:21:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4939.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:21:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4940.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:21:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4941.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:21:58 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:21:58 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.4935.0> 2011-07-14 00:21:59 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4935.0> 2011-07-14 00:21:59 =============================================================================== ** Generic server <5901.4935.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.43489>}, init, {1310,617318,900311}, "default",#Port<5901.3010>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4935.0> 2011-07-14 00:21:59 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4935.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4909.0>] messages [] links [<5901.49.0>,<5901.4910.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.4910.0> 2011-07-14 00:21:59 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4935.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:21:59 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 208 times in the past 206.65512 secs (last seen 0.999334 secs ago PROGRESS REPORT <5901.4910.0> 2011-07-14 00:21:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4947.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:21:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4948.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:21:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4949.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:21:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4950.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:21:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4951.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:21:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4952.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4947.0> 2011-07-14 00:21:59 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4947.0> 2011-07-14 00:21:59 =============================================================================== ** Generic server <5901.4947.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.43616>}, init, {1310,617319,401768}, "default",#Port<5901.3012>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4947.0> 2011-07-14 00:21:59 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4947.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4909.0>] messages [] links [<5901.49.0>,<5901.4910.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4910.0> 2011-07-14 00:21:59 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4947.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4949.0> 2011-07-14 00:21:59 =============================================================================== ns_1@10.88.165.188:<5901.4949.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:21:59 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 214 times in the past 206.998735 secs (last seen 1.001396 secs ago PROGRESS REPORT <5901.4910.0> 2011-07-14 00:21:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4959.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:21:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4960.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:21:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4961.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:21:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4962.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:21:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4963.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:21:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4964.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4959.0> 2011-07-14 00:22:00 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4959.0> 2011-07-14 00:22:00 =============================================================================== ** Generic server <5901.4959.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.43703>}, init, {1310,617319,901766}, "default",#Port<5901.3013>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4959.0> 2011-07-14 00:22:00 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4959.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4909.0>] messages [] links [<5901.49.0>,<5901.4910.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4910.0> 2011-07-14 00:22:00 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4959.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:00 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 209 times in the past 207.658654 secs (last seen 1.003534 secs ago PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4968.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:00 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:00 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4969.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4970.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4971.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4972.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4973.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4968.0> 2011-07-14 00:22:00 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4968.0> 2011-07-14 00:22:00 =============================================================================== ** Generic server <5901.4968.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.43790>}, init, {1310,617320,405239}, "default",#Port<5901.3014>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4968.0> 2011-07-14 00:22:00 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4968.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4909.0>] messages [] links [<5901.49.0>,<5901.4910.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4910.0> 2011-07-14 00:22:00 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4968.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4970.0> 2011-07-14 00:22:00 =============================================================================== ns_1@10.88.165.188:<5901.4970.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:00 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 215 times in the past 207.998902 secs (last seen 1.000167 secs ago PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4978.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4979.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4980.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4981.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4982.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4983.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4978.0> 2011-07-14 00:22:01 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4978.0> 2011-07-14 00:22:01 =============================================================================== ** Generic server <5901.4978.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.43888>}, init, {1310,617320,902027}, "default",#Port<5901.3015>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4978.0> 2011-07-14 00:22:01 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4978.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4909.0>] messages [] links [<5901.49.0>,<5901.4910.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4910.0> 2011-07-14 00:22:01 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4978.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:01 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 210 times in the past 208.6588 secs (last seen 1.000146 secs ago PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4987.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:01 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:01 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4988.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4989.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4990.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4991.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4992.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4987.0> 2011-07-14 00:22:01 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4987.0> 2011-07-14 00:22:01 =============================================================================== ** Generic server <5901.4987.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.43974>}, init, {1310,617321,405354}, "default",#Port<5901.3016>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4987.0> 2011-07-14 00:22:01 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4987.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4909.0>] messages [] links [<5901.49.0>,<5901.4910.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4910.0> 2011-07-14 00:22:01 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4987.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.4989.0> 2011-07-14 00:22:01 =============================================================================== ns_1@10.88.165.188:<5901.4989.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:01 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 216 times in the past 208.998988 secs (last seen 1.000086 secs ago PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.4999.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5000.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5001.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5002.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5003.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5004.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.4999.0> 2011-07-14 00:22:02 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.4999.0> 2011-07-14 00:22:02 =============================================================================== ** Generic server <5901.4999.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.44057>}, init, {1310,617321,902188}, "default",#Port<5901.3017>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.4999.0> 2011-07-14 00:22:02 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.4999.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4909.0>] messages [] links [<5901.49.0>,<5901.4910.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4910.0> 2011-07-14 00:22:02 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.4999.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:02 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 211 times in the past 209.658919 secs (last seen 1.000119 secs ago PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5009.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5010.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5011.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:02 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:02 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5012.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5013.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5014.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5009.0> 2011-07-14 00:22:02 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5009.0> 2011-07-14 00:22:02 =============================================================================== ** Generic server <5901.5009.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.44187>}, init, {1310,617322,405495}, "default",#Port<5901.3019>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5009.0> 2011-07-14 00:22:02 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5009.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4909.0>] messages [] links [<5901.49.0>,<5901.4910.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4910.0> 2011-07-14 00:22:02 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5009.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5011.0> 2011-07-14 00:22:02 =============================================================================== ns_1@10.88.165.188:<5901.5011.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:02 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 217 times in the past 209.999084 secs (last seen 1.000096 secs ago PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5018.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5019.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5020.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5021.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5022.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:02 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5023.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5018.0> 2011-07-14 00:22:03 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5018.0> 2011-07-14 00:22:03 =============================================================================== ** Generic server <5901.5018.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.44268>}, init, {1310,617322,902338}, "default",#Port<5901.3020>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5018.0> 2011-07-14 00:22:03 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5018.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4909.0>] messages [] links [<5901.49.0>,<5901.4910.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4910.0> 2011-07-14 00:22:03 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5018.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:03 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 212 times in the past 210.658976 secs (last seen 1.000057 secs ago PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5027.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5028.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5029.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5030.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5031.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5032.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5027.0> 2011-07-14 00:22:03 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5027.0> 2011-07-14 00:22:03 =============================================================================== ** Generic server <5901.5027.0> terminating ** Last message in was list_vbuckets_prevstate ** When Server state == {state,{interval,#Ref<5901.0.0.44352>}, init, {1310,617323,405586}, "default",#Port<5901.3021>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5027.0> 2011-07-14 00:22:03 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5027.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4909.0>] messages [{'$gen_call',{<5901.5029.0>,#Ref<5901.0.0.44457>},{stats,<<>>}}] links [<5901.49.0>,<5901.4910.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6671 SUPERVISOR REPORT <5901.4910.0> 2011-07-14 00:22:03 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5027.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker CRASH REPORT <5901.5042.0> 2011-07-14 00:22:03 =============================================================================== Crashing process initial_call {ns_janitor,cleanup,['Argument__1','Argument__2']} pid <5901.5042.0> registered_name [] error_info {exit,{{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, {gen_server,call, [{'ns_memcached-default','ns_1@10.88.165.188'}, list_vbuckets_prevstate,30000]}}, [{gen_server,call,3}, {ns_janitor,'-read_existing_map/4-lc$^0/1-0-',2}, {ns_janitor,read_existing_map,4}, {ns_janitor,cleanup,2}, {proc_lib,init_p_do_apply,3}]} ancestors [<5901.304.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<5901.52.0>] messages [] links [<5901.304.0>] dictionary [] trap_exit false status running heap_size 10946 stack_size 24 reductions 224 ERROR REPORT <5901.5029.0> 2011-07-14 00:22:03 =============================================================================== ns_1@10.88.165.188:<5901.5029.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:03 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 218 times in the past 210.997343 secs (last seen 0.998259 secs ago INFO REPORT <5901.304.0> 2011-07-14 00:22:03 =============================================================================== ns_1@10.88.165.188:<5901.304.0>:ns_orchestrator:178: Janitor run exited for bucket "default" with reason {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, [{'ns_memcached-default', 'ns_1@10.88.165.188'}, list_vbuckets_prevstate, 30000]}} PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5045.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5046.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5048.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:03 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:03 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5049.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5051.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:03 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5052.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5045.0> 2011-07-14 00:22:04 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5045.0> 2011-07-14 00:22:04 =============================================================================== ** Generic server <5901.5045.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.44467>}, init, {1310,617323,900818}, "default",#Port<5901.3023>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5045.0> 2011-07-14 00:22:04 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5045.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4909.0>] messages [] links [<5901.49.0>,<5901.4910.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.4910.0> 2011-07-14 00:22:04 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5045.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:04 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 213 times in the past 211.655108 secs (last seen 0.996132 secs ago PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5057.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5058.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5059.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5060.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5061.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5062.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5057.0> 2011-07-14 00:22:04 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5057.0> 2011-07-14 00:22:04 =============================================================================== ** Generic server <5901.5057.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.44583>}, init, {1310,617324,401765}, "default",#Port<5901.3024>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5057.0> 2011-07-14 00:22:04 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5057.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4909.0>] messages [check_started] links [<5901.49.0>,<5901.4910.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4910.0> 2011-07-14 00:22:04 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5057.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5059.0> 2011-07-14 00:22:04 =============================================================================== ns_1@10.88.165.188:<5901.5059.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:04 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 219 times in the past 212.000384 secs (last seen 1.003041 secs ago PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5066.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5067.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5068.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5069.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5070.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:04 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5071.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5066.0> 2011-07-14 00:22:05 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5066.0> 2011-07-14 00:22:05 =============================================================================== ** Generic server <5901.5066.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.44661>}, init, {1310,617324,903504}, "default",#Port<5901.3025>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5066.0> 2011-07-14 00:22:05 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5066.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4909.0>] messages [] links [<5901.49.0>,<5901.4910.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4910.0> 2011-07-14 00:22:05 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5066.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:05 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 214 times in the past 212.659071 secs (last seen 1.003963 secs ago PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5076.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:05 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:05 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5077.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5078.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5079.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5080.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5081.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5076.0> 2011-07-14 00:22:05 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5076.0> 2011-07-14 00:22:05 =============================================================================== ** Generic server <5901.5076.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.44784>}, init, {1310,617325,405702}, "default",#Port<5901.3027>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5076.0> 2011-07-14 00:22:05 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5076.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4909.0>] messages [] links [<5901.49.0>,<5901.4910.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4910.0> 2011-07-14 00:22:05 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5076.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5078.0> 2011-07-14 00:22:05 =============================================================================== ns_1@10.88.165.188:<5901.5078.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:05 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 220 times in the past 212.999095 secs (last seen 0.998711 secs ago PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5088.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5089.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5090.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5091.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5092.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:05 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5093.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5088.0> 2011-07-14 00:22:06 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5088.0> 2011-07-14 00:22:06 =============================================================================== ** Generic server <5901.5088.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.44867>}, init, {1310,617325,902198}, "default",#Port<5901.3028>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5088.0> 2011-07-14 00:22:06 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5088.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4909.0>] messages [] links [<5901.49.0>,<5901.4910.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4910.0> 2011-07-14 00:22:06 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5088.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:06 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 215 times in the past 213.659028 secs (last seen 0.999957 secs ago PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5097.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5098.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5099.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:06 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:06 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5100.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5101.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5102.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5097.0> 2011-07-14 00:22:06 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5097.0> 2011-07-14 00:22:06 =============================================================================== ** Generic server <5901.5097.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.44953>}, init, {1310,617326,405603}, "default",#Port<5901.3029>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5097.0> 2011-07-14 00:22:06 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5097.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4909.0>] messages [] links [<5901.49.0>,<5901.4910.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4910.0> 2011-07-14 00:22:06 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5097.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5099.0> 2011-07-14 00:22:06 =============================================================================== ns_1@10.88.165.188:<5901.5099.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:06 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 221 times in the past 213.999356 secs (last seen 1.000261 secs ago PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5106.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5107.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5108.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5109.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5110.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:06 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5111.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5106.0> 2011-07-14 00:22:07 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5106.0> 2011-07-14 00:22:07 =============================================================================== ** Generic server <5901.5106.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.45033>}, init, {1310,617326,903452}, "default",#Port<5901.3030>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5106.0> 2011-07-14 00:22:07 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5106.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4909.0>] messages [] links [<5901.49.0>,<5901.4910.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4910.0> 2011-07-14 00:22:07 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5106.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:07 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 216 times in the past 214.659101 secs (last seen 1.000073 secs ago PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5116.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5117.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5118.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5119.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5120.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5121.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5116.0> 2011-07-14 00:22:07 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5116.0> 2011-07-14 00:22:07 =============================================================================== ** Generic server <5901.5116.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.45137>}, init, {1310,617327,405685}, "default",#Port<5901.3031>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5116.0> 2011-07-14 00:22:07 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5116.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4909.0>] messages [] links [<5901.49.0>,<5901.4910.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.4910.0> 2011-07-14 00:22:07 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5116.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5118.0> 2011-07-14 00:22:07 =============================================================================== ns_1@10.88.165.188:<5901.5118.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:07 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 222 times in the past 214.999225 secs (last seen 0.999869 secs ago PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5128.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5129.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5130.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5131.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5132.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.4910.0> 2011-07-14 00:22:07 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5133.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:07 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:07 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.5128.0> 2011-07-14 00:22:08 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5128.0> 2011-07-14 00:22:08 =============================================================================== ** Generic server <5901.5128.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.45223>}, init, {1310,617327,902332}, "default",#Port<5901.3032>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5128.0> 2011-07-14 00:22:08 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5128.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.4909.0>] messages [] links [<5901.49.0>,<5901.4910.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.4910.0> 2011-07-14 00:22:08 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5128.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker SUPERVISOR REPORT <5901.4910.0> 2011-07-14 00:22:08 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext shutdown reason reached_max_restart_intensity pid <5901.5128.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:08 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 217 times in the past 215.656319 secs (last seen 0.997218 secs ago INFO REPORT <5901.4909.0> 2011-07-14 00:22:08 =============================================================================== ns_1@10.88.165.188:<5901.4909.0>:single_bucket_sup:24: per-bucket supervisor for "default" died with reason shutdown SUPERVISOR REPORT <5901.273.0> 2011-07-14 00:22:08 =============================================================================== Reporting supervisor {local,ns_bucket_sup} Child process errorContext child_terminated reason shutdown pid <5901.4909.0> name {per_bucket_sup,"default"} mfargs {single_bucket_sup,start_link,["default"]} restart_type permanent shutdown infinity child_type supervisor PROGRESS REPORT <5901.273.0> 2011-07-14 00:22:08 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<5901.5138.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5140.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5141.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5142.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5143.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5144.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5145.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5140.0> 2011-07-14 00:22:08 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5140.0> 2011-07-14 00:22:08 =============================================================================== ** Generic server <5901.5140.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.45349>}, init, {1310,617328,406089}, "default",#Port<5901.3034>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5140.0> 2011-07-14 00:22:08 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5140.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5138.0>] messages [] links [<5901.49.0>,<5901.5139.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6665 SUPERVISOR REPORT <5901.5139.0> 2011-07-14 00:22:08 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5140.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5142.0> 2011-07-14 00:22:08 =============================================================================== ns_1@10.88.165.188:<5901.5142.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:08 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 223 times in the past 215.999383 secs (last seen 1.000158 secs ago PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5152.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5153.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5155.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5156.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5157.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:08 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5158.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5152.0> 2011-07-14 00:22:09 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5152.0> 2011-07-14 00:22:09 =============================================================================== ** Generic server <5901.5152.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.45443>}, init, {1310,617328,902482}, "default",#Port<5901.3035>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5152.0> 2011-07-14 00:22:09 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5152.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5138.0>] messages [] links [<5901.49.0>,<5901.5139.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.5139.0> 2011-07-14 00:22:09 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5152.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:09 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 218 times in the past 216.65656 secs (last seen 1.000241 secs ago PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5163.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5164.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5165.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:09 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:09 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5166.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5167.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5168.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5163.0> 2011-07-14 00:22:09 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5163.0> 2011-07-14 00:22:09 =============================================================================== ** Generic server <5901.5163.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.45538>}, init, {1310,617329,405261}, "default",#Port<5901.3036>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5163.0> 2011-07-14 00:22:09 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5163.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5138.0>] messages [] links [<5901.49.0>,<5901.5139.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5139.0> 2011-07-14 00:22:09 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5163.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5165.0> 2011-07-14 00:22:09 =============================================================================== ns_1@10.88.165.188:<5901.5165.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:09 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 224 times in the past 216.999907 secs (last seen 1.000524 secs ago PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5175.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5176.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5177.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5178.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5179.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:09 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5180.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5175.0> 2011-07-14 00:22:10 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5175.0> 2011-07-14 00:22:10 =============================================================================== ** Generic server <5901.5175.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.45619>}, init, {1310,617329,903104}, "default",#Port<5901.3037>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5175.0> 2011-07-14 00:22:10 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5175.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5138.0>] messages [] links [<5901.49.0>,<5901.5139.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5139.0> 2011-07-14 00:22:10 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5175.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:10 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 219 times in the past 217.65943 secs (last seen 1.00287 secs ago PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5185.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5186.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5187.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5188.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5189.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5190.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5185.0> 2011-07-14 00:22:10 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5185.0> 2011-07-14 00:22:10 =============================================================================== ** Generic server <5901.5185.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.45728>}, init, {1310,617330,406054}, "default",#Port<5901.3038>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5185.0> 2011-07-14 00:22:10 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5185.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5138.0>] messages [] links [<5901.49.0>,<5901.5139.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5139.0> 2011-07-14 00:22:10 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5185.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5187.0> 2011-07-14 00:22:10 =============================================================================== ns_1@10.88.165.188:<5901.5187.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:10 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 225 times in the past 217.999492 secs (last seen 0.999585 secs ago PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5194.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5195.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5196.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:10 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:10 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5197.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5198.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:10 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5199.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5194.0> 2011-07-14 00:22:11 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5194.0> 2011-07-14 00:22:11 =============================================================================== ** Generic server <5901.5194.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.45808>}, init, {1310,617330,902621}, "default",#Port<5901.3039>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5194.0> 2011-07-14 00:22:11 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5194.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5138.0>] messages [] links [<5901.49.0>,<5901.5139.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5139.0> 2011-07-14 00:22:11 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5194.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:11 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 220 times in the past 218.656683 secs (last seen 0.997253 secs ago PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5204.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5205.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5206.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5207.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5208.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5209.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5204.0> 2011-07-14 00:22:11 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5204.0> 2011-07-14 00:22:11 =============================================================================== ** Generic server <5901.5204.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.45926>}, init, {1310,617331,403322}, "default",#Port<5901.3041>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5204.0> 2011-07-14 00:22:11 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5204.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5138.0>] messages [] links [<5901.49.0>,<5901.5139.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5139.0> 2011-07-14 00:22:11 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5204.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5206.0> 2011-07-14 00:22:11 =============================================================================== ns_1@10.88.165.188:<5901.5206.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:11 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 226 times in the past 218.999471 secs (last seen 0.999979 secs ago PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5216.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5217.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5218.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5219.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5220.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:11 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5221.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5216.0> 2011-07-14 00:22:12 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5216.0> 2011-07-14 00:22:12 =============================================================================== ** Generic server <5901.5216.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.46013>}, init, {1310,617331,903479}, "default",#Port<5901.3042>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5216.0> 2011-07-14 00:22:12 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5216.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5138.0>] messages [] links [<5901.49.0>,<5901.5139.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5139.0> 2011-07-14 00:22:12 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5216.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:12 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 221 times in the past 219.656634 secs (last seen 0.999951 secs ago PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5225.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:12 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:12 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5226.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5227.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5228.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5229.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5230.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5225.0> 2011-07-14 00:22:12 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5225.0> 2011-07-14 00:22:12 =============================================================================== ** Generic server <5901.5225.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.46100>}, init, {1310,617332,403173}, "default",#Port<5901.3043>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5225.0> 2011-07-14 00:22:12 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5225.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5138.0>] messages [] links [<5901.49.0>,<5901.5139.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5139.0> 2011-07-14 00:22:12 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5225.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5227.0> 2011-07-14 00:22:12 =============================================================================== ns_1@10.88.165.188:<5901.5227.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:12 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 227 times in the past 219.999667 secs (last seen 1.000196 secs ago PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5234.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5235.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5236.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5237.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5238.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:12 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5239.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5234.0> 2011-07-14 00:22:13 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5234.0> 2011-07-14 00:22:13 =============================================================================== ** Generic server <5901.5234.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.46180>}, init, {1310,617332,902809}, "default",#Port<5901.3044>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5234.0> 2011-07-14 00:22:13 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5234.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5138.0>] messages [] links [<5901.49.0>,<5901.5139.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5139.0> 2011-07-14 00:22:13 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5234.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:13 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 222 times in the past 220.659535 secs (last seen 1.002901 secs ago PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5243.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5244.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5245.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5246.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5247.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5248.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.256.0> 2011-07-14 00:22:13 =============================================================================== ns_1@10.88.165.188:<5901.256.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.88.165.188', [{last_heard,{1310,617333,603343}}, {active_buckets,["default"]}, {ready_buckets,[]}, {replication,[{"default",1.0}]}, {memory, [{total,28904408}, {processes,7099016}, {processes_used,7090232}, {system,21805392}, {atom,814593}, {atom_used,805810}, {binary,507112}, {code,7689836}, {ets,3474008}]}, {system_stats,[{cpu_utilization_rate,2.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,241}, {memory_data,{644874240,585121792,{<5901.80.0>,601112}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 58724 kB\nBuffers: 22352 kB\nCached: 367988 kB\nSwapCached: 0 kB\nActive: 222820 kB\nInactive: 270996 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 58724 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 452 kB\nWriteback: 0 kB\nAnonPages: 103456 kB\nMapped: 12304 kB\nSlab: 36964 kB\nPageTables: 4164 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 552372 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,376819712}, {buffered_memory,22888448}, {free_memory,60133376}, {total_memory,644874240}]}, {statistics, [{wall_clock,{226637,1}}, {context_switches,{118995,0}}, {garbage_collection,{25378,52539678,0}}, {io,{{input,16946931},{output,10151888}}}, {reductions,{13857311,878419}}, {run_queue,0}, {runtime,{820,20}}]}]}, {'ns_1@10.88.23.62', [{last_heard,{1310,617333,613427}}, {active_buckets,["default"]}, {ready_buckets,[]}, {replication,[{"default",1.0}]}, {memory, [{total,30148296}, {processes,7846400}, {processes_used,7820096}, {system,22301896}, {atom,820249}, {atom_used,814132}, {binary,932192}, {code,7741308}, {ets,3486872}]}, {system_stats,[{cpu_utilization_rate,0.0},{swap_total,0},{swap_used,0}]}, {interesting_stats, [{curr_items,0},{curr_items_tot,0},{vb_replica_curr_items,0}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.14.3"}, {mnesia,"4.4.17"}, {stdlib,"1.17.3"}, {os_mon,"2.2.5"}, {ns_server,"1.7.1r-42-gd713cca"}, {sasl,"2.1.9.3"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,245}, {memory_data,{644874240,589692928,{<5990.256.0>,830584}}}, {disk_data,[{"/",5904748,66},{"/dev/shm",314880,0}]}, {meminfo, <<"MemTotal: 629760 kB\nMemFree: 53888 kB\nBuffers: 21880 kB\nCached: 369840 kB\nSwapCached: 0 kB\nActive: 228392 kB\nInactive: 270140 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 629760 kB\nLowFree: 53888 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nDirty: 676 kB\nWriteback: 0 kB\nAnonPages: 106828 kB\nMapped: 12316 kB\nSlab: 37100 kB\nPageTables: 4116 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 314880 kB\nCommitted_AS: 554696 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 240 kB\nVmallocChunk: 34359738127 kB\n">>}, {system_memory_data, [{system_total_memory,644874240}, {free_swap,0}, {total_swap,0}, {cached_memory,378716160}, {buffered_memory,22405120}, {free_memory,55181312}, {total_memory,644874240}]}, {statistics, [{wall_clock,{241505,1}}, {context_switches,{137263,0}}, {garbage_collection,{27851,76131621,0}}, {io,{{input,17858283},{output,10471581}}}, {reductions,{16222502,894484}}, {run_queue,0}, {runtime,{1280,60}}]}]}] INFO REPORT <5901.5243.0> 2011-07-14 00:22:13 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5243.0> 2011-07-14 00:22:13 =============================================================================== ** Generic server <5901.5243.0> terminating ** Last message in was list_vbuckets_prevstate ** When Server state == {state,{interval,#Ref<5901.0.0.46268>}, init, {1310,617333,406102}, "default",#Port<5901.3045>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5243.0> 2011-07-14 00:22:13 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5243.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5138.0>] messages [] links [<5901.49.0>,<5901.5139.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6671 SUPERVISOR REPORT <5901.5139.0> 2011-07-14 00:22:13 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5243.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker CRASH REPORT <5901.5257.0> 2011-07-14 00:22:13 =============================================================================== Crashing process initial_call {ns_janitor,cleanup,['Argument__1','Argument__2']} pid <5901.5257.0> registered_name [] error_info {exit,{{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, {gen_server,call, [{'ns_memcached-default','ns_1@10.88.165.188'}, list_vbuckets_prevstate,30000]}}, [{gen_server,call,3}, {ns_janitor,'-read_existing_map/4-lc$^0/1-0-',2}, {ns_janitor,read_existing_map,4}, {ns_janitor,cleanup,2}, {proc_lib,init_p_do_apply,3}]} ancestors [<5901.304.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<5901.52.0>] messages [] links [<5901.304.0>] dictionary [] trap_exit false status running heap_size 10946 stack_size 24 reductions 224 INFO REPORT <5901.227.0> 2011-07-14 00:22:13 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 228 times in the past 220.997372 secs (last seen 0.997705 secs ago INFO REPORT <5901.304.0> 2011-07-14 00:22:13 =============================================================================== ns_1@10.88.165.188:<5901.304.0>:ns_orchestrator:178: Janitor run exited for bucket "default" with reason {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, [{'ns_memcached-default', 'ns_1@10.88.165.188'}, list_vbuckets_prevstate, 30000]}} ERROR REPORT <5901.5245.0> 2011-07-14 00:22:13 =============================================================================== ns_1@10.88.165.188:<5901.5245.0>:stats_collector:94: Exception in stats collector: {exit, {noproc, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5264.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5265.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5267.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5268.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5269.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:13 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5270.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:13 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:13 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.5264.0> 2011-07-14 00:22:14 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5264.0> 2011-07-14 00:22:14 =============================================================================== ** Generic server <5901.5264.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.46478>}, init, {1310,617333,900772}, "default",#Port<5901.3048>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5264.0> 2011-07-14 00:22:14 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5264.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5138.0>] messages [] links [<5901.49.0>,<5901.5139.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.5139.0> 2011-07-14 00:22:14 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5264.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:14 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 223 times in the past 221.655541 secs (last seen 0.996006 secs ago PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5276.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5277.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5278.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5279.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5280.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5281.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.239.0> 2011-07-14 00:22:14 =============================================================================== ns_1@10.88.165.188:<5901.239.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.5276.0> 2011-07-14 00:22:14 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5276.0> 2011-07-14 00:22:14 =============================================================================== ** Generic server <5901.5276.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.46622>}, init, {1310,617334,402154}, "default",#Port<5901.3050>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5276.0> 2011-07-14 00:22:14 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5276.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5138.0>] messages [] links [<5901.49.0>,<5901.5139.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5139.0> 2011-07-14 00:22:14 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5276.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5278.0> 2011-07-14 00:22:14 =============================================================================== ns_1@10.88.165.188:<5901.5278.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:14 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 229 times in the past 221.999773 secs (last seen 1.002401 secs ago PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5287.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5288.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5289.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5290.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5291.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:14 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5292.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5287.0> 2011-07-14 00:22:15 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5287.0> 2011-07-14 00:22:15 =============================================================================== ** Generic server <5901.5287.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.46714>}, init, {1310,617334,902795}, "default",#Port<5901.3052>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5287.0> 2011-07-14 00:22:15 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5287.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5138.0>] messages [] links [<5901.49.0>,<5901.5139.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5139.0> 2011-07-14 00:22:15 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5287.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:15 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 224 times in the past 222.657287 secs (last seen 1.001746 secs ago PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5296.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:15 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:15 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5297.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5298.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5299.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5300.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5301.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5296.0> 2011-07-14 00:22:15 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5296.0> 2011-07-14 00:22:15 =============================================================================== ** Generic server <5901.5296.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.46796>}, init, {1310,617335,407939}, "default",#Port<5901.3053>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5296.0> 2011-07-14 00:22:15 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5296.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5138.0>] messages [] links [<5901.49.0>,<5901.5139.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5139.0> 2011-07-14 00:22:15 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5296.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5298.0> 2011-07-14 00:22:15 =============================================================================== ns_1@10.88.165.188:<5901.5298.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:15 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 230 times in the past 222.999745 secs (last seen 0.999972 secs ago PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5308.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5309.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5310.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5311.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5312.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:15 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5313.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5308.0> 2011-07-14 00:22:16 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5308.0> 2011-07-14 00:22:16 =============================================================================== ** Generic server <5901.5308.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.46883>}, init, {1310,617335,902969}, "default",#Port<5901.3054>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5308.0> 2011-07-14 00:22:16 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5308.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5138.0>] messages [] links [<5901.49.0>,<5901.5139.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5139.0> 2011-07-14 00:22:16 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5308.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:16 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 225 times in the past 223.657127 secs (last seen 0.99984 secs ago PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5317.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5318.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5319.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5320.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5321.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5322.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:16 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:16 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.5317.0> 2011-07-14 00:22:16 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5317.0> 2011-07-14 00:22:16 =============================================================================== ** Generic server <5901.5317.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.46969>}, init, {1310,617336,404550}, "default",#Port<5901.3055>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5317.0> 2011-07-14 00:22:16 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5317.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5138.0>] messages [] links [<5901.49.0>,<5901.5139.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5139.0> 2011-07-14 00:22:16 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5317.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5319.0> 2011-07-14 00:22:16 =============================================================================== ns_1@10.88.165.188:<5901.5319.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:16 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 231 times in the past 223.99988 secs (last seen 1.000135 secs ago PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5327.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5328.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5329.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5330.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5331.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:16 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5332.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5327.0> 2011-07-14 00:22:17 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5327.0> 2011-07-14 00:22:17 =============================================================================== ** Generic server <5901.5327.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.47065>}, init, {1310,617336,903034}, "default",#Port<5901.3056>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5327.0> 2011-07-14 00:22:17 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5327.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5138.0>] messages [] links [<5901.49.0>,<5901.5139.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5139.0> 2011-07-14 00:22:17 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5327.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:17 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 226 times in the past 224.659768 secs (last seen 1.002641 secs ago PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5337.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5338.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5339.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5340.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5341.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5342.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5337.0> 2011-07-14 00:22:17 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5337.0> 2011-07-14 00:22:17 =============================================================================== ** Generic server <5901.5337.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.47190>}, init, {1310,617337,406408}, "default",#Port<5901.3058>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5337.0> 2011-07-14 00:22:17 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5337.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5138.0>] messages [] links [<5901.49.0>,<5901.5139.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5139.0> 2011-07-14 00:22:17 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5337.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5339.0> 2011-07-14 00:22:17 =============================================================================== ns_1@10.88.165.188:<5901.5339.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:17 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 232 times in the past 224.999837 secs (last seen 0.999957 secs ago PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5349.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:17 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:17 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5350.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5351.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5352.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5353.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:17 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5354.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5349.0> 2011-07-14 00:22:18 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5349.0> 2011-07-14 00:22:18 =============================================================================== ** Generic server <5901.5349.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.47276>}, init, {1310,617337,903174}, "default",#Port<5901.3059>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5349.0> 2011-07-14 00:22:18 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5349.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5138.0>] messages [] links [<5901.49.0>,<5901.5139.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5139.0> 2011-07-14 00:22:18 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5349.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:18 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 227 times in the past 225.659832 secs (last seen 1.000064 secs ago PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5358.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5359.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5360.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5361.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5362.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5139.0> 2011-07-14 00:22:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5363.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5358.0> 2011-07-14 00:22:18 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5358.0> 2011-07-14 00:22:18 =============================================================================== ** Generic server <5901.5358.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.47364>}, init, {1310,617338,406491}, "default",#Port<5901.3060>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5358.0> 2011-07-14 00:22:18 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5358.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5138.0>] messages [] links [<5901.49.0>,<5901.5139.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6665 SUPERVISOR REPORT <5901.5139.0> 2011-07-14 00:22:18 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5358.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker SUPERVISOR REPORT <5901.5139.0> 2011-07-14 00:22:18 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext shutdown reason reached_max_restart_intensity pid <5901.5358.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5360.0> 2011-07-14 00:22:18 =============================================================================== ns_1@10.88.165.188:<5901.5360.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:18 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 233 times in the past 225.999839 secs (last seen 1.000002 secs ago INFO REPORT <5901.5138.0> 2011-07-14 00:22:18 =============================================================================== ns_1@10.88.165.188:<5901.5138.0>:single_bucket_sup:24: per-bucket supervisor for "default" died with reason shutdown SUPERVISOR REPORT <5901.273.0> 2011-07-14 00:22:18 =============================================================================== Reporting supervisor {local,ns_bucket_sup} Child process errorContext child_terminated reason shutdown pid <5901.5138.0> name {per_bucket_sup,"default"} mfargs {single_bucket_sup,start_link,["default"]} restart_type permanent shutdown infinity child_type supervisor PROGRESS REPORT <5901.273.0> 2011-07-14 00:22:18 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<5901.5370.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5372.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:18 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:18 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5373.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5375.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5376.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5378.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:18 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5379.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5372.0> 2011-07-14 00:22:19 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5372.0> 2011-07-14 00:22:19 =============================================================================== ** Generic server <5901.5372.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.47460>}, init, {1310,617338,904013}, "default",#Port<5901.3061>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5372.0> 2011-07-14 00:22:19 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5372.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5370.0>] messages [] links [<5901.49.0>,<5901.5371.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.5371.0> 2011-07-14 00:22:19 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5372.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:19 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 228 times in the past 226.657065 secs (last seen 0.997233 secs ago PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5383.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5384.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5385.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5386.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5387.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5388.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5383.0> 2011-07-14 00:22:19 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5383.0> 2011-07-14 00:22:19 =============================================================================== ** Generic server <5901.5383.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.47553>}, init, {1310,617339,403718}, "default",#Port<5901.3062>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5383.0> 2011-07-14 00:22:19 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5383.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5370.0>] messages [] links [<5901.49.0>,<5901.5371.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5371.0> 2011-07-14 00:22:19 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5383.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5385.0> 2011-07-14 00:22:19 =============================================================================== ns_1@10.88.165.188:<5901.5385.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:19 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 234 times in the past 226.99723 secs (last seen 0.997391 secs ago PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5396.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5397.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5398.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5399.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5400.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:19 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5401.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:19 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:19 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.5396.0> 2011-07-14 00:22:20 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5396.0> 2011-07-14 00:22:20 =============================================================================== ** Generic server <5901.5396.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.47659>}, init, {1310,617339,900152}, "default",#Port<5901.3063>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5396.0> 2011-07-14 00:22:20 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5396.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5370.0>] messages [] links [<5901.49.0>,<5901.5371.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5371.0> 2011-07-14 00:22:20 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5396.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:20 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 229 times in the past 227.65687 secs (last seen 0.999805 secs ago PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5406.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5407.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5408.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5409.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5410.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5411.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5406.0> 2011-07-14 00:22:20 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5406.0> 2011-07-14 00:22:20 =============================================================================== ** Generic server <5901.5406.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.47785>}, init, {1310,617340,403557}, "default",#Port<5901.3065>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5406.0> 2011-07-14 00:22:20 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5406.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5370.0>] messages [check_started] links [<5901.49.0>,<5901.5371.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5371.0> 2011-07-14 00:22:20 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5406.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5408.0> 2011-07-14 00:22:20 =============================================================================== ns_1@10.88.165.188:<5901.5408.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:20 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 235 times in the past 228.002132 secs (last seen 1.004902 secs ago PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5415.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5416.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5417.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5418.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5419.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:20 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5420.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5415.0> 2011-07-14 00:22:21 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5415.0> 2011-07-14 00:22:21 =============================================================================== ** Generic server <5901.5415.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.47863>}, init, {1310,617340,905235}, "default",#Port<5901.3066>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5415.0> 2011-07-14 00:22:21 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5415.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5370.0>] messages [] links [<5901.49.0>,<5901.5371.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5371.0> 2011-07-14 00:22:21 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5415.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:21 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 230 times in the past 228.660941 secs (last seen 1.004071 secs ago PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5424.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:21 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:21 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5425.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5426.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5427.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5428.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5429.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5424.0> 2011-07-14 00:22:21 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5424.0> 2011-07-14 00:22:21 =============================================================================== ** Generic server <5901.5424.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.47949>}, init, {1310,617341,407600}, "default",#Port<5901.3067>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5424.0> 2011-07-14 00:22:21 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5424.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5370.0>] messages [] links [<5901.49.0>,<5901.5371.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5371.0> 2011-07-14 00:22:21 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5424.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5426.0> 2011-07-14 00:22:21 =============================================================================== ns_1@10.88.165.188:<5901.5426.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:21 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 236 times in the past 228.997321 secs (last seen 0.995189 secs ago PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5436.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5437.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5438.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5439.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5440.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:21 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5441.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5436.0> 2011-07-14 00:22:22 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5436.0> 2011-07-14 00:22:22 =============================================================================== ** Generic server <5901.5436.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.48032>}, init, {1310,617341,900644}, "default",#Port<5901.3068>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5436.0> 2011-07-14 00:22:22 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5436.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5370.0>] messages [] links [<5901.49.0>,<5901.5371.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5371.0> 2011-07-14 00:22:22 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5436.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:22 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 231 times in the past 229.65691 secs (last seen 0.995969 secs ago PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5445.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5446.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5447.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5448.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5449.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5450.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:22 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:22 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.5445.0> 2011-07-14 00:22:22 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5445.0> 2011-07-14 00:22:22 =============================================================================== ** Generic server <5901.5445.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.48119>}, init, {1310,617342,403474}, "default",#Port<5901.3069>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5445.0> 2011-07-14 00:22:22 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5445.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5370.0>] messages [] links [<5901.49.0>,<5901.5371.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5371.0> 2011-07-14 00:22:22 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5445.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5447.0> 2011-07-14 00:22:22 =============================================================================== ns_1@10.88.165.188:<5901.5447.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:22 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 237 times in the past 229.997235 secs (last seen 0.999914 secs ago PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5455.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5456.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5457.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5458.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5459.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:22 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5460.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5455.0> 2011-07-14 00:22:23 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5455.0> 2011-07-14 00:22:23 =============================================================================== ** Generic server <5901.5455.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.48219>}, init, {1310,617342,900405}, "default",#Port<5901.3070>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5455.0> 2011-07-14 00:22:23 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5455.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5370.0>] messages [] links [<5901.49.0>,<5901.5371.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5371.0> 2011-07-14 00:22:23 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5455.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:23 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 232 times in the past 230.657038 secs (last seen 1.000128 secs ago PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5465.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5466.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5467.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5468.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5469.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5470.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] ERROR REPORT <5901.5467.0> 2011-07-14 00:22:23 =============================================================================== ns_1@10.88.165.188:<5901.5467.0>:stats_collector:94: Exception in stats collector: {exit, {noproc, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.5465.0> 2011-07-14 00:22:23 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5465.0> 2011-07-14 00:22:23 =============================================================================== ** Generic server <5901.5465.0> terminating ** Last message in was list_vbuckets_prevstate ** When Server state == {state,{interval,#Ref<5901.0.0.48339>}, init, {1310,617343,403611}, "default",#Port<5901.3072>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5465.0> 2011-07-14 00:22:23 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5465.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5370.0>] messages [] links [<5901.49.0>,<5901.5371.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6671 SUPERVISOR REPORT <5901.5371.0> 2011-07-14 00:22:23 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5465.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker CRASH REPORT <5901.5477.0> 2011-07-14 00:22:23 =============================================================================== Crashing process initial_call {ns_janitor,cleanup,['Argument__1','Argument__2']} pid <5901.5477.0> registered_name [] error_info {exit,{{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, {gen_server,call, [{'ns_memcached-default','ns_1@10.88.165.188'}, list_vbuckets_prevstate,30000]}}, [{gen_server,call,3}, {ns_janitor,'-read_existing_map/4-lc$^0/1-0-',2}, {ns_janitor,read_existing_map,4}, {ns_janitor,cleanup,2}, {proc_lib,init_p_do_apply,3}]} ancestors [<5901.304.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<5901.52.0>] messages [] links [<5901.304.0>] dictionary [] trap_exit false status running heap_size 10946 stack_size 24 reductions 224 INFO REPORT <5901.227.0> 2011-07-14 00:22:23 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 238 times in the past 230.997057 secs (last seen 0.999822 secs ago INFO REPORT <5901.304.0> 2011-07-14 00:22:23 =============================================================================== ns_1@10.88.165.188:<5901.304.0>:ns_orchestrator:178: Janitor run exited for bucket "default" with reason {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, [{'ns_memcached-default', 'ns_1@10.88.165.188'}, list_vbuckets_prevstate, 30000]}} PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5483.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:23 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:23 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5484.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5486.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5487.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5488.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:23 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5489.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5483.0> 2011-07-14 00:22:24 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5483.0> 2011-07-14 00:22:24 =============================================================================== ** Generic server <5901.5483.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.48453>}, init, {1310,617343,900945}, "default",#Port<5901.3074>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5483.0> 2011-07-14 00:22:24 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5483.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5370.0>] messages [] links [<5901.49.0>,<5901.5371.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.5371.0> 2011-07-14 00:22:24 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5483.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:24 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 233 times in the past 231.654155 secs (last seen 0.997117 secs ago PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5494.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5495.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5496.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5497.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5498.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5499.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5494.0> 2011-07-14 00:22:24 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5494.0> 2011-07-14 00:22:24 =============================================================================== ** Generic server <5901.5494.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.48552>}, init, {1310,617344,400724}, "default",#Port<5901.3075>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5494.0> 2011-07-14 00:22:24 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5494.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5370.0>] messages [] links [<5901.49.0>,<5901.5371.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5371.0> 2011-07-14 00:22:24 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5494.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5496.0> 2011-07-14 00:22:24 =============================================================================== ns_1@10.88.165.188:<5901.5496.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:24 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 239 times in the past 231.997287 secs (last seen 1.00023 secs ago PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5503.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5504.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5505.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:24 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:24 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5506.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5507.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:24 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5508.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5503.0> 2011-07-14 00:22:25 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5503.0> 2011-07-14 00:22:25 =============================================================================== ** Generic server <5901.5503.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.48628>}, init, {1310,617344,900335}, "default",#Port<5901.3076>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5503.0> 2011-07-14 00:22:25 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5503.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5370.0>] messages [] links [<5901.49.0>,<5901.5371.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5371.0> 2011-07-14 00:22:25 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5503.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:25 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 234 times in the past 232.654475 secs (last seen 1.00032 secs ago PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5512.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5513.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5514.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5515.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5516.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5517.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5512.0> 2011-07-14 00:22:25 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5512.0> 2011-07-14 00:22:25 =============================================================================== ** Generic server <5901.5512.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.48714>}, init, {1310,617345,401043}, "default",#Port<5901.3077>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5512.0> 2011-07-14 00:22:25 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5512.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5370.0>] messages [] links [<5901.49.0>,<5901.5371.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5371.0> 2011-07-14 00:22:25 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5512.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5514.0> 2011-07-14 00:22:25 =============================================================================== ns_1@10.88.165.188:<5901.5514.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:25 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 240 times in the past 232.998202 secs (last seen 1.000915 secs ago PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5524.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5525.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5526.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5527.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5528.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:25 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5529.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5524.0> 2011-07-14 00:22:26 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5524.0> 2011-07-14 00:22:26 =============================================================================== ** Generic server <5901.5524.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.48797>}, init, {1310,617345,901859}, "default",#Port<5901.3078>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5524.0> 2011-07-14 00:22:26 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5524.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5370.0>] messages [] links [<5901.49.0>,<5901.5371.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5371.0> 2011-07-14 00:22:26 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5524.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:26 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 235 times in the past 233.657156 secs (last seen 1.002681 secs ago PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5535.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:26 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:26 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5536.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5537.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5538.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5539.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5540.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5535.0> 2011-07-14 00:22:26 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5535.0> 2011-07-14 00:22:26 =============================================================================== ** Generic server <5901.5535.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.48947>}, init, {1310,617346,403665}, "default",#Port<5901.3080>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5535.0> 2011-07-14 00:22:26 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5535.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5370.0>] messages [] links [<5901.49.0>,<5901.5371.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5371.0> 2011-07-14 00:22:26 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5535.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5537.0> 2011-07-14 00:22:26 =============================================================================== ns_1@10.88.165.188:<5901.5537.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:26 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 241 times in the past 233.997385 secs (last seen 0.999183 secs ago PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5544.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5545.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5546.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5547.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5548.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:26 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5549.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5544.0> 2011-07-14 00:22:27 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5544.0> 2011-07-14 00:22:27 =============================================================================== ** Generic server <5901.5544.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.49030>}, init, {1310,617346,900405}, "default",#Port<5901.3081>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5544.0> 2011-07-14 00:22:27 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5544.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5370.0>] messages [] links [<5901.49.0>,<5901.5371.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5371.0> 2011-07-14 00:22:27 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5544.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:27 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 236 times in the past 234.65453 secs (last seen 0.997374 secs ago PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5553.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5554.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5555.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5556.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5557.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5558.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:27 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:27 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.5553.0> 2011-07-14 00:22:27 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5553.0> 2011-07-14 00:22:27 =============================================================================== ** Generic server <5901.5553.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.49112>}, init, {1310,617347,401055}, "default",#Port<5901.3082>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5553.0> 2011-07-14 00:22:27 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5553.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5370.0>] messages [] links [<5901.49.0>,<5901.5371.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5371.0> 2011-07-14 00:22:27 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5553.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5555.0> 2011-07-14 00:22:27 =============================================================================== ns_1@10.88.165.188:<5901.5555.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:27 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 242 times in the past 234.997323 secs (last seen 0.999938 secs ago PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5565.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5566.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5567.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5568.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5569.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:27 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5570.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.239.0> 2011-07-14 00:22:28 =============================================================================== ns_1@10.88.165.188:<5901.239.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.5565.0> 2011-07-14 00:22:28 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5565.0> 2011-07-14 00:22:28 =============================================================================== ** Generic server <5901.5565.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.49197>}, init, {1310,617347,900306}, "default",#Port<5901.3083>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5565.0> 2011-07-14 00:22:28 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5565.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5370.0>] messages [] links [<5901.49.0>,<5901.5371.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5371.0> 2011-07-14 00:22:28 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5565.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:28 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 237 times in the past 235.654342 secs (last seen 0.999812 secs ago PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5576.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5577.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5578.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5579.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5580.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5581.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5576.0> 2011-07-14 00:22:28 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5576.0> 2011-07-14 00:22:28 =============================================================================== ** Generic server <5901.5576.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.49294>}, init, {1310,617348,400872}, "default",#Port<5901.3085>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5576.0> 2011-07-14 00:22:28 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5576.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5370.0>] messages [] links [<5901.49.0>,<5901.5371.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6665 SUPERVISOR REPORT <5901.5371.0> 2011-07-14 00:22:28 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5576.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5578.0> 2011-07-14 00:22:28 =============================================================================== ns_1@10.88.165.188:<5901.5578.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:28 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 243 times in the past 235.99738 secs (last seen 1.000057 secs ago PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5589.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:28 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:28 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5590.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5592.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5593.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5595.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5371.0> 2011-07-14 00:22:28 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5596.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5589.0> 2011-07-14 00:22:29 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5589.0> 2011-07-14 00:22:29 =============================================================================== ** Generic server <5901.5589.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.49463>}, init, {1310,617348,900312}, "default",#Port<5901.3087>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5589.0> 2011-07-14 00:22:29 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5589.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5370.0>] messages [] links [<5901.49.0>,<5901.5371.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.5371.0> 2011-07-14 00:22:29 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5589.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker SUPERVISOR REPORT <5901.5371.0> 2011-07-14 00:22:29 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext shutdown reason reached_max_restart_intensity pid <5901.5589.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:29 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 238 times in the past 236.654586 secs (last seen 1.000244 secs ago INFO REPORT <5901.5370.0> 2011-07-14 00:22:29 =============================================================================== ns_1@10.88.165.188:<5901.5370.0>:single_bucket_sup:24: per-bucket supervisor for "default" died with reason shutdown SUPERVISOR REPORT <5901.273.0> 2011-07-14 00:22:29 =============================================================================== Reporting supervisor {local,ns_bucket_sup} Child process errorContext child_terminated reason shutdown pid <5901.5370.0> name {per_bucket_sup,"default"} mfargs {single_bucket_sup,start_link,["default"]} restart_type permanent shutdown infinity child_type supervisor PROGRESS REPORT <5901.273.0> 2011-07-14 00:22:29 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<5901.5602.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5604.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5605.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5606.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5607.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5608.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5609.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5604.0> 2011-07-14 00:22:29 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5604.0> 2011-07-14 00:22:29 =============================================================================== ** Generic server <5901.5604.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.49617>}, init, {1310,617349,401574}, "default",#Port<5901.3089>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5604.0> 2011-07-14 00:22:29 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5604.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5602.0>] messages [] links [<5901.49.0>,<5901.5603.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5603.0> 2011-07-14 00:22:29 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5604.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5606.0> 2011-07-14 00:22:29 =============================================================================== ns_1@10.88.165.188:<5901.5606.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:29 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 244 times in the past 236.997535 secs (last seen 1.000155 secs ago PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5616.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5617.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5618.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:29 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:29 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5619.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5620.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:29 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5621.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5616.0> 2011-07-14 00:22:30 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5616.0> 2011-07-14 00:22:30 =============================================================================== ** Generic server <5901.5616.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.49698>}, init, {1310,617349,900664}, "default",#Port<5901.3090>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5616.0> 2011-07-14 00:22:30 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5616.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5602.0>] messages [] links [<5901.49.0>,<5901.5603.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5603.0> 2011-07-14 00:22:30 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5616.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:30 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 239 times in the past 237.654721 secs (last seen 1.000135 secs ago PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5625.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5626.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5627.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5628.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5629.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5630.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5625.0> 2011-07-14 00:22:30 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5625.0> 2011-07-14 00:22:30 =============================================================================== ** Generic server <5901.5625.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.49784>}, init, {1310,617350,401296}, "default",#Port<5901.3091>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5625.0> 2011-07-14 00:22:30 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5625.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5602.0>] messages [] links [<5901.49.0>,<5901.5603.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5603.0> 2011-07-14 00:22:30 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5625.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5627.0> 2011-07-14 00:22:30 =============================================================================== ns_1@10.88.165.188:<5901.5627.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:30 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 245 times in the past 237.997616 secs (last seen 1.000081 secs ago PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5634.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5635.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5636.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5637.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5638.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:30 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5639.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5634.0> 2011-07-14 00:22:31 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5634.0> 2011-07-14 00:22:31 =============================================================================== ** Generic server <5901.5634.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.49866>}, init, {1310,617350,901488}, "default",#Port<5901.3092>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5634.0> 2011-07-14 00:22:31 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5634.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5602.0>] messages [] links [<5901.49.0>,<5901.5603.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5603.0> 2011-07-14 00:22:31 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5634.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:31 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 240 times in the past 238.654846 secs (last seen 1.000125 secs ago PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5643.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5644.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5645.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:31 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:31 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5646.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5647.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5648.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5643.0> 2011-07-14 00:22:31 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5643.0> 2011-07-14 00:22:31 =============================================================================== ** Generic server <5901.5643.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.49949>}, init, {1310,617351,401389}, "default",#Port<5901.3093>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5643.0> 2011-07-14 00:22:31 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5643.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5602.0>] messages [] links [<5901.49.0>,<5901.5603.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5603.0> 2011-07-14 00:22:31 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5643.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5645.0> 2011-07-14 00:22:31 =============================================================================== ns_1@10.88.165.188:<5901.5645.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:31 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 246 times in the past 238.997583 secs (last seen 0.999967 secs ago PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5655.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5656.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5657.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5658.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5659.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:31 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5660.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5655.0> 2011-07-14 00:22:32 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5655.0> 2011-07-14 00:22:32 =============================================================================== ** Generic server <5901.5655.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.50035>}, init, {1310,617351,900596}, "default",#Port<5901.3096>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5655.0> 2011-07-14 00:22:32 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5655.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5602.0>] messages [] links [<5901.49.0>,<5901.5603.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5603.0> 2011-07-14 00:22:32 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5655.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:32 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 241 times in the past 239.654588 secs (last seen 0.999742 secs ago PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5666.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5667.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5668.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5669.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5670.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5671.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5666.0> 2011-07-14 00:22:32 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5666.0> 2011-07-14 00:22:32 =============================================================================== ** Generic server <5901.5666.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.50183>}, init, {1310,617352,401088}, "default",#Port<5901.3098>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5666.0> 2011-07-14 00:22:32 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5666.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5602.0>] messages [] links [<5901.49.0>,<5901.5603.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5603.0> 2011-07-14 00:22:32 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5666.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5668.0> 2011-07-14 00:22:32 =============================================================================== ns_1@10.88.165.188:<5901.5668.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:32 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 247 times in the past 239.997773 secs (last seen 1.00019 secs ago PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5675.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:32 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:32 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5676.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5677.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5678.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5679.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:32 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5680.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5675.0> 2011-07-14 00:22:33 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5675.0> 2011-07-14 00:22:33 =============================================================================== ** Generic server <5901.5675.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.50259>}, init, {1310,617352,901736}, "default",#Port<5901.3099>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5675.0> 2011-07-14 00:22:33 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5675.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5602.0>] messages [] links [<5901.49.0>,<5901.5603.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5603.0> 2011-07-14 00:22:33 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5675.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:33 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 242 times in the past 240.65491 secs (last seen 1.000322 secs ago PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5684.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5685.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5686.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5687.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5688.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5689.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5684.0> 2011-07-14 00:22:33 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5684.0> 2011-07-14 00:22:33 =============================================================================== ** Generic server <5901.5684.0> terminating ** Last message in was list_vbuckets_prevstate ** When Server state == {state,{interval,#Ref<5901.0.0.50348>}, init, {1310,617353,401464}, "default",#Port<5901.3100>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5684.0> 2011-07-14 00:22:33 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5684.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5602.0>] messages [] links [<5901.49.0>,<5901.5603.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6671 SUPERVISOR REPORT <5901.5603.0> 2011-07-14 00:22:33 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5684.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker CRASH REPORT <5901.5696.0> 2011-07-14 00:22:33 =============================================================================== Crashing process initial_call {ns_janitor,cleanup,['Argument__1','Argument__2']} pid <5901.5696.0> registered_name [] error_info {exit,{{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, {gen_server,call, [{'ns_memcached-default','ns_1@10.88.165.188'}, list_vbuckets_prevstate,30000]}}, [{gen_server,call,3}, {ns_janitor,'-read_existing_map/4-lc$^0/1-0-',2}, {ns_janitor,read_existing_map,4}, {ns_janitor,cleanup,2}, {proc_lib,init_p_do_apply,3}]} ancestors [<5901.304.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<5901.52.0>] messages [] links [<5901.304.0>] dictionary [] trap_exit false status running heap_size 10946 stack_size 24 reductions 224 INFO REPORT <5901.227.0> 2011-07-14 00:22:33 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 248 times in the past 240.995854 secs (last seen 0.998081 secs ago INFO REPORT <5901.304.0> 2011-07-14 00:22:33 =============================================================================== ns_1@10.88.165.188:<5901.304.0>:ns_orchestrator:178: Janitor run exited for bucket "default" with reason {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, [{'ns_memcached-default', 'ns_1@10.88.165.188'}, list_vbuckets_prevstate, 30000]}} ERROR REPORT <5901.5686.0> 2011-07-14 00:22:33 =============================================================================== ns_1@10.88.165.188:<5901.5686.0>:stats_collector:94: Exception in stats collector: {exit, {noproc, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5702.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5703.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5704.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5705.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5708.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:33 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5709.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:33 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:33 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.5702.0> 2011-07-14 00:22:34 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5702.0> 2011-07-14 00:22:34 =============================================================================== ** Generic server <5901.5702.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.50458>}, init, {1310,617353,899465}, "default",#Port<5901.3102>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5702.0> 2011-07-14 00:22:34 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5702.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5602.0>] messages [] links [<5901.49.0>,<5901.5603.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.5603.0> 2011-07-14 00:22:34 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5702.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:34 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 243 times in the past 241.65406 secs (last seen 0.99915 secs ago PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5713.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5714.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5715.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5716.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5717.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5718.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5713.0> 2011-07-14 00:22:34 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5713.0> 2011-07-14 00:22:34 =============================================================================== ** Generic server <5901.5713.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.50554>}, init, {1310,617354,400681}, "default",#Port<5901.3103>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5713.0> 2011-07-14 00:22:34 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5713.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5602.0>] messages [] links [<5901.49.0>,<5901.5603.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5603.0> 2011-07-14 00:22:34 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5713.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5715.0> 2011-07-14 00:22:34 =============================================================================== ns_1@10.88.165.188:<5901.5715.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:34 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 249 times in the past 241.9979 secs (last seen 1.002046 secs ago PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5722.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5723.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5724.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5725.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5726.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:34 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5727.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5722.0> 2011-07-14 00:22:35 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5722.0> 2011-07-14 00:22:35 =============================================================================== ** Generic server <5901.5722.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.50636>}, init, {1310,617354,900876}, "default",#Port<5901.3104>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5722.0> 2011-07-14 00:22:35 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5722.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5602.0>] messages [] links [<5901.49.0>,<5901.5603.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5603.0> 2011-07-14 00:22:35 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5722.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:35 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 244 times in the past 242.657759 secs (last seen 1.003699 secs ago PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5732.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:35 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:35 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5733.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5734.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5735.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5736.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5737.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5732.0> 2011-07-14 00:22:35 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5732.0> 2011-07-14 00:22:35 =============================================================================== ** Generic server <5901.5732.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.50755>}, init, {1310,617355,404310}, "default",#Port<5901.3107>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5732.0> 2011-07-14 00:22:35 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5732.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5602.0>] messages [] links [<5901.49.0>,<5901.5603.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5603.0> 2011-07-14 00:22:35 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5732.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5734.0> 2011-07-14 00:22:35 =============================================================================== ns_1@10.88.165.188:<5901.5734.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:35 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 250 times in the past 242.997855 secs (last seen 0.999955 secs ago PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5745.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5746.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5747.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5748.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5749.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:35 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5750.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5745.0> 2011-07-14 00:22:36 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5745.0> 2011-07-14 00:22:36 =============================================================================== ** Generic server <5901.5745.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.50863>}, init, {1310,617355,901032}, "default",#Port<5901.3108>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5745.0> 2011-07-14 00:22:36 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5745.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5602.0>] messages [] links [<5901.49.0>,<5901.5603.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5603.0> 2011-07-14 00:22:36 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5745.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:36 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 245 times in the past 243.657827 secs (last seen 1.000068 secs ago PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5754.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:36 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:36 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5755.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5756.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5757.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5758.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5759.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5754.0> 2011-07-14 00:22:36 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5754.0> 2011-07-14 00:22:36 =============================================================================== ** Generic server <5901.5754.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.50949>}, init, {1310,617356,404430}, "default",#Port<5901.3109>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5754.0> 2011-07-14 00:22:36 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5754.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5602.0>] messages [] links [<5901.49.0>,<5901.5603.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5603.0> 2011-07-14 00:22:36 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5754.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5756.0> 2011-07-14 00:22:36 =============================================================================== ns_1@10.88.165.188:<5901.5756.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:36 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 251 times in the past 243.999762 secs (last seen 1.001907 secs ago PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5763.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5764.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5765.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5766.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5767.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:36 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5768.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5763.0> 2011-07-14 00:22:37 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5763.0> 2011-07-14 00:22:37 =============================================================================== ** Generic server <5901.5763.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.51027>}, init, {1310,617356,902841}, "default",#Port<5901.3110>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5763.0> 2011-07-14 00:22:37 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5763.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5602.0>] messages [] links [<5901.49.0>,<5901.5603.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5603.0> 2011-07-14 00:22:37 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5763.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:37 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 246 times in the past 244.657921 secs (last seen 1.000094 secs ago PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5772.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5773.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5774.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:37 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:37 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5775.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5776.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5777.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5772.0> 2011-07-14 00:22:37 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5772.0> 2011-07-14 00:22:37 =============================================================================== ** Generic server <5901.5772.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.51113>}, init, {1310,617357,404477}, "default",#Port<5901.3111>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5772.0> 2011-07-14 00:22:37 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5772.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5602.0>] messages [] links [<5901.49.0>,<5901.5603.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5603.0> 2011-07-14 00:22:37 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5772.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5774.0> 2011-07-14 00:22:37 =============================================================================== ns_1@10.88.165.188:<5901.5774.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:37 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 252 times in the past 244.998032 secs (last seen 0.99827 secs ago PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5784.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5785.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5786.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5787.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5788.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:37 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5789.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5784.0> 2011-07-14 00:22:38 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5784.0> 2011-07-14 00:22:38 =============================================================================== ** Generic server <5901.5784.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.51194>}, init, {1310,617357,901108}, "default",#Port<5901.3112>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5784.0> 2011-07-14 00:22:38 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5784.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5602.0>] messages [] links [<5901.49.0>,<5901.5603.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5603.0> 2011-07-14 00:22:38 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5784.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:38 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 247 times in the past 245.658023 secs (last seen 1.000102 secs ago PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5794.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5795.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5796.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5797.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5798.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5799.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5794.0> 2011-07-14 00:22:38 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5794.0> 2011-07-14 00:22:38 =============================================================================== ** Generic server <5901.5794.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.51327>}, init, {1310,617358,404633}, "default",#Port<5901.3114>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5794.0> 2011-07-14 00:22:38 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5794.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5602.0>] messages [] links [<5901.49.0>,<5901.5603.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6665 SUPERVISOR REPORT <5901.5603.0> 2011-07-14 00:22:38 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5794.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5796.0> 2011-07-14 00:22:38 =============================================================================== ns_1@10.88.165.188:<5901.5796.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:38 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 253 times in the past 245.998021 secs (last seen 0.999989 secs ago PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5807.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:38 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:38 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5808.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5810.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5811.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5813.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:38 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5814.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5807.0> 2011-07-14 00:22:39 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5807.0> 2011-07-14 00:22:39 =============================================================================== ** Generic server <5901.5807.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.51443>}, init, {1310,617358,900924}, "default",#Port<5901.3115>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5807.0> 2011-07-14 00:22:39 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5807.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5602.0>] messages [] links [<5901.49.0>,<5901.5603.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.5603.0> 2011-07-14 00:22:39 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5807.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:39 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 248 times in the past 246.655204 secs (last seen 0.997181 secs ago PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5818.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5819.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5820.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5821.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5822.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5603.0> 2011-07-14 00:22:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5823.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5818.0> 2011-07-14 00:22:39 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5818.0> 2011-07-14 00:22:39 =============================================================================== ** Generic server <5901.5818.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.51534>}, init, {1310,617359,401858}, "default",#Port<5901.3116>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5818.0> 2011-07-14 00:22:39 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5818.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5602.0>] messages [] links [<5901.49.0>,<5901.5603.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5603.0> 2011-07-14 00:22:39 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5818.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker SUPERVISOR REPORT <5901.5603.0> 2011-07-14 00:22:39 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext shutdown reason reached_max_restart_intensity pid <5901.5818.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5820.0> 2011-07-14 00:22:39 =============================================================================== ns_1@10.88.165.188:<5901.5820.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:39 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 254 times in the past 246.998171 secs (last seen 1.00015 secs ago INFO REPORT <5901.5602.0> 2011-07-14 00:22:39 =============================================================================== ns_1@10.88.165.188:<5901.5602.0>:single_bucket_sup:24: per-bucket supervisor for "default" died with reason shutdown SUPERVISOR REPORT <5901.273.0> 2011-07-14 00:22:39 =============================================================================== Reporting supervisor {local,ns_bucket_sup} Child process errorContext child_terminated reason shutdown pid <5901.5602.0> name {per_bucket_sup,"default"} mfargs {single_bucket_sup,start_link,["default"]} restart_type permanent shutdown infinity child_type supervisor PROGRESS REPORT <5901.273.0> 2011-07-14 00:22:39 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<5901.5830.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5832.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5833.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5834.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5835.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5836.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:39 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5837.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5832.0> 2011-07-14 00:22:40 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5832.0> 2011-07-14 00:22:40 =============================================================================== ** Generic server <5901.5832.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.51621>}, init, {1310,617359,902592}, "default",#Port<5901.3117>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5832.0> 2011-07-14 00:22:40 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5832.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5830.0>] messages [] links [<5901.49.0>,<5901.5831.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5831.0> 2011-07-14 00:22:40 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5832.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:40 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 249 times in the past 247.655463 secs (last seen 1.000259 secs ago PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5841.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:40 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:40 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5842.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5843.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5844.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5845.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5846.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5841.0> 2011-07-14 00:22:40 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5841.0> 2011-07-14 00:22:40 =============================================================================== ** Generic server <5901.5841.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.51708>}, init, {1310,617360,402041}, "default",#Port<5901.3118>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5841.0> 2011-07-14 00:22:40 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5841.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5830.0>] messages [] links [<5901.49.0>,<5901.5831.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5831.0> 2011-07-14 00:22:40 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5841.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5843.0> 2011-07-14 00:22:40 =============================================================================== ns_1@10.88.165.188:<5901.5843.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:40 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 255 times in the past 247.998389 secs (last seen 1.000218 secs ago PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5850.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5851.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5852.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5853.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5854.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:40 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5855.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5850.0> 2011-07-14 00:22:41 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5850.0> 2011-07-14 00:22:41 =============================================================================== ** Generic server <5901.5850.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.51786>}, init, {1310,617360,901516}, "default",#Port<5901.3119>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5850.0> 2011-07-14 00:22:41 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5850.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5830.0>] messages [] links [<5901.49.0>,<5901.5831.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5831.0> 2011-07-14 00:22:41 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5850.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:41 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 250 times in the past 248.658278 secs (last seen 1.002815 secs ago PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5860.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:41 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:41 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5861.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5862.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5863.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5864.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5865.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5860.0> 2011-07-14 00:22:41 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5860.0> 2011-07-14 00:22:41 =============================================================================== ** Generic server <5901.5860.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.51908>}, init, {1310,617361,404876}, "default",#Port<5901.3121>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5860.0> 2011-07-14 00:22:41 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5860.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5830.0>] messages [] links [<5901.49.0>,<5901.5831.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5831.0> 2011-07-14 00:22:41 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5860.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5862.0> 2011-07-14 00:22:41 =============================================================================== ns_1@10.88.165.188:<5901.5862.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:41 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 256 times in the past 248.998238 secs (last seen 0.999849 secs ago PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5873.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5874.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5875.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5876.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5877.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:41 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5878.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5873.0> 2011-07-14 00:22:42 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5873.0> 2011-07-14 00:22:42 =============================================================================== ** Generic server <5901.5873.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.52011>}, init, {1310,617361,901250}, "default",#Port<5901.3122>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5873.0> 2011-07-14 00:22:42 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5873.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5830.0>] messages [] links [<5901.49.0>,<5901.5831.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5831.0> 2011-07-14 00:22:42 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5873.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:42 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 251 times in the past 249.655298 secs (last seen 0.99702 secs ago PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5882.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5883.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5884.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5885.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5886.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5887.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:42 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:42 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.5882.0> 2011-07-14 00:22:42 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5882.0> 2011-07-14 00:22:42 =============================================================================== ** Generic server <5901.5882.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.52098>}, init, {1310,617362,401770}, "default",#Port<5901.3123>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5882.0> 2011-07-14 00:22:42 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5882.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5830.0>] messages [] links [<5901.49.0>,<5901.5831.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5831.0> 2011-07-14 00:22:42 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5882.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5884.0> 2011-07-14 00:22:42 =============================================================================== ns_1@10.88.165.188:<5901.5884.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:42 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 257 times in the past 249.998398 secs (last seen 1.00016 secs ago PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5891.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5892.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5893.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5894.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5895.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:42 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5896.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5891.0> 2011-07-14 00:22:43 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5891.0> 2011-07-14 00:22:43 =============================================================================== ** Generic server <5901.5891.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.52183>}, init, {1310,617362,901514}, "default",#Port<5901.3124>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5891.0> 2011-07-14 00:22:43 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5891.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5830.0>] messages [] links [<5901.49.0>,<5901.5831.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5831.0> 2011-07-14 00:22:43 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5891.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:43 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 252 times in the past 250.658318 secs (last seen 1.00302 secs ago PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5900.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5901.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5902.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5903.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5904.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5905.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5900.0> 2011-07-14 00:22:43 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5900.0> 2011-07-14 00:22:43 =============================================================================== ** Generic server <5901.5900.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.52267>}, init, {1310,617363,404926}, "default",#Port<5901.3125>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5900.0> 2011-07-14 00:22:43 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5900.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5830.0>] messages [] links [<5901.49.0>,<5901.5831.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6671 SUPERVISOR REPORT <5901.5831.0> 2011-07-14 00:22:43 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5900.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5902.0> 2011-07-14 00:22:43 =============================================================================== ns_1@10.88.165.188:<5901.5902.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:43 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 258 times in the past 250.997242 secs (last seen 0.998844 secs ago PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5918.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:43 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:43 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5920.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5922.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5923.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5925.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5926.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5918.0> 2011-07-14 00:22:43 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5918.0> 2011-07-14 00:22:43 =============================================================================== ** Generic server <5901.5918.0> terminating ** Last message in was list_vbuckets_prevstate ** When Server state == {state,{interval,#Ref<5901.0.0.52382>}, init, {1310,617363,900751}, "default",#Port<5901.3128>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5918.0> 2011-07-14 00:22:43 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5918.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5830.0>] messages [] links [<5901.49.0>,<5901.5831.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6701 SUPERVISOR REPORT <5901.5831.0> 2011-07-14 00:22:43 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5918.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker CRASH REPORT <5901.5912.0> 2011-07-14 00:22:43 =============================================================================== Crashing process initial_call {ns_janitor,cleanup,['Argument__1','Argument__2']} pid <5901.5912.0> registered_name [] error_info {exit,{{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, {gen_server,call, [{'ns_memcached-default','ns_1@10.88.165.188'}, list_vbuckets_prevstate,30000]}}, [{gen_server,call,3}, {ns_janitor,'-read_existing_map/4-lc$^0/1-0-',2}, {ns_janitor,read_existing_map,4}, {ns_janitor,cleanup,2}, {proc_lib,init_p_do_apply,3}]} ancestors [<5901.304.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<5901.52.0>] messages [] links [<5901.304.0>] dictionary [] trap_exit false status running heap_size 10946 stack_size 24 reductions 224 INFO REPORT <5901.227.0> 2011-07-14 00:22:43 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 259 times in the past 251.009209 secs (last seen 0.011967 secs ago INFO REPORT <5901.304.0> 2011-07-14 00:22:43 =============================================================================== ns_1@10.88.165.188:<5901.304.0>:ns_orchestrator:178: Janitor run exited for bucket "default" with reason {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, [{'ns_memcached-default', 'ns_1@10.88.165.188'}, list_vbuckets_prevstate, 30000]}} PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5927.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5928.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5929.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5930.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5931.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:43 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5933.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5927.0> 2011-07-14 00:22:44 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5927.0> 2011-07-14 00:22:44 =============================================================================== ** Generic server <5901.5927.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.52533>}, init, {1310,617363,911388}, "default",#Port<5901.3129>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5927.0> 2011-07-14 00:22:44 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5927.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5830.0>] messages [] links [<5901.49.0>,<5901.5831.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5831.0> 2011-07-14 00:22:44 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5927.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:44 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 253 times in the past 251.666326 secs (last seen 1.008008 secs ago PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5937.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5938.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5939.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5940.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5941.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5942.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5937.0> 2011-07-14 00:22:44 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5937.0> 2011-07-14 00:22:44 =============================================================================== ** Generic server <5901.5937.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.52656>}, init, {1310,617364,412937}, "default",#Port<5901.3131>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5937.0> 2011-07-14 00:22:44 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5937.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5830.0>] messages [] links [<5901.49.0>,<5901.5831.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5831.0> 2011-07-14 00:22:44 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5937.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5939.0> 2011-07-14 00:22:44 =============================================================================== ns_1@10.88.165.188:<5901.5939.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:44 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 260 times in the past 251.998392 secs (last seen 0.989183 secs ago PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5946.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5947.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5948.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5949.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5950.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:44 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5951.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:44 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:44 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.239.0> 2011-07-14 00:22:45 =============================================================================== ns_1@10.88.165.188:<5901.239.0>:ns_config_rep:257: Pulling config from: 'ns_1@10.88.23.62' INFO REPORT <5901.5946.0> 2011-07-14 00:22:45 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5946.0> 2011-07-14 00:22:45 =============================================================================== ** Generic server <5901.5946.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.52732>}, init, {1310,617364,901411}, "default",#Port<5901.3132>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5946.0> 2011-07-14 00:22:45 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5946.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5830.0>] messages [] links [<5901.49.0>,<5901.5831.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5831.0> 2011-07-14 00:22:45 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5946.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:45 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 254 times in the past 252.655534 secs (last seen 0.989208 secs ago PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5958.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5959.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5960.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5961.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5962.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5963.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5958.0> 2011-07-14 00:22:45 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5958.0> 2011-07-14 00:22:45 =============================================================================== ** Generic server <5901.5958.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.52847>}, init, {1310,617365,402063}, "default",#Port<5901.3134>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5958.0> 2011-07-14 00:22:45 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5958.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5830.0>] messages [] links [<5901.49.0>,<5901.5831.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5831.0> 2011-07-14 00:22:45 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5958.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5960.0> 2011-07-14 00:22:45 =============================================================================== ns_1@10.88.165.188:<5901.5960.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:45 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 261 times in the past 252.998494 secs (last seen 1.000102 secs ago PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5970.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5971.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5972.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5973.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5974.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:45 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5975.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5970.0> 2011-07-14 00:22:46 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5970.0> 2011-07-14 00:22:46 =============================================================================== ** Generic server <5901.5970.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.52930>}, init, {1310,617365,901563}, "default",#Port<5901.3135>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5970.0> 2011-07-14 00:22:46 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5970.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5830.0>] messages [] links [<5901.49.0>,<5901.5831.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5831.0> 2011-07-14 00:22:46 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5970.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:46 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 255 times in the past 253.655507 secs (last seen 0.999973 secs ago PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5979.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5980.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:46 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:46 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5981.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5982.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5983.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5984.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5979.0> 2011-07-14 00:22:46 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5979.0> 2011-07-14 00:22:46 =============================================================================== ** Generic server <5901.5979.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.53016>}, init, {1310,617366,401949}, "default",#Port<5901.3136>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5979.0> 2011-07-14 00:22:46 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5979.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5830.0>] messages [] links [<5901.49.0>,<5901.5831.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5831.0> 2011-07-14 00:22:46 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5979.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.5981.0> 2011-07-14 00:22:46 =============================================================================== ns_1@10.88.165.188:<5901.5981.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:46 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 262 times in the past 253.99858 secs (last seen 1.000086 secs ago PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5988.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5989.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5990.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5991.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5992.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:46 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5993.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5988.0> 2011-07-14 00:22:47 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5988.0> 2011-07-14 00:22:47 =============================================================================== ** Generic server <5901.5988.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.53097>}, init, {1310,617366,901661}, "default",#Port<5901.3137>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5988.0> 2011-07-14 00:22:47 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5988.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5830.0>] messages [] links [<5901.49.0>,<5901.5831.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5831.0> 2011-07-14 00:22:47 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5988.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:47 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 256 times in the past 254.655787 secs (last seen 1.00028 secs ago PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5998.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.5999.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6000.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6001.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6002.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6003.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.5998.0> 2011-07-14 00:22:47 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.5998.0> 2011-07-14 00:22:47 =============================================================================== ** Generic server <5901.5998.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.53217>}, init, {1310,617367,402326}, "default",#Port<5901.3139>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.5998.0> 2011-07-14 00:22:47 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.5998.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5830.0>] messages [] links [<5901.49.0>,<5901.5831.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5831.0> 2011-07-14 00:22:47 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.5998.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.6000.0> 2011-07-14 00:22:47 =============================================================================== ns_1@10.88.165.188:<5901.6000.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:47 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 263 times in the past 254.99865 secs (last seen 1.00007 secs ago PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6010.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:47 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:47 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6011.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6012.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6013.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6014.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:47 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6015.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.6010.0> 2011-07-14 00:22:48 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.6010.0> 2011-07-14 00:22:48 =============================================================================== ** Generic server <5901.6010.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.53302>}, init, {1310,617367,901629}, "default",#Port<5901.3140>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.6010.0> 2011-07-14 00:22:48 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.6010.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5830.0>] messages [] links [<5901.49.0>,<5901.5831.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.5831.0> 2011-07-14 00:22:48 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.6010.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:48 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 257 times in the past 255.655677 secs (last seen 0.99989 secs ago PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6020.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6021.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6022.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6023.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6024.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6025.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.6020.0> 2011-07-14 00:22:48 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.6020.0> 2011-07-14 00:22:48 =============================================================================== ** Generic server <5901.6020.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.53410>}, init, {1310,617368,402181}, "default",#Port<5901.3141>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.6020.0> 2011-07-14 00:22:48 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.6020.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5830.0>] messages [] links [<5901.49.0>,<5901.5831.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6665 SUPERVISOR REPORT <5901.5831.0> 2011-07-14 00:22:48 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.6020.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.6022.0> 2011-07-14 00:22:48 =============================================================================== ns_1@10.88.165.188:<5901.6022.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:48 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 264 times in the past 255.998645 secs (last seen 0.999995 secs ago INFO REPORT <5901.269.0> 2011-07-14 00:22:48 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:48 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6032.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6034.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6035.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6036.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6037.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:48 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6038.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.6032.0> 2011-07-14 00:22:49 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.6032.0> 2011-07-14 00:22:49 =============================================================================== ** Generic server <5901.6032.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.53502>}, init, {1310,617368,902064}, "default",#Port<5901.3142>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.6032.0> 2011-07-14 00:22:49 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.6032.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5830.0>] messages [] links [<5901.49.0>,<5901.5831.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.5831.0> 2011-07-14 00:22:49 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.6032.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:49 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 258 times in the past 256.655713 secs (last seen 1.000036 secs ago PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6043.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6044.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6045.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6046.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6047.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.5831.0> 2011-07-14 00:22:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6048.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.6043.0> 2011-07-14 00:22:49 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.6043.0> 2011-07-14 00:22:49 =============================================================================== ** Generic server <5901.6043.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.53598>}, init, {1310,617369,402251}, "default",#Port<5901.3143>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.6043.0> 2011-07-14 00:22:49 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.6043.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.5830.0>] messages [] links [<5901.49.0>,<5901.5831.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.5831.0> 2011-07-14 00:22:49 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.6043.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker SUPERVISOR REPORT <5901.5831.0> 2011-07-14 00:22:49 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext shutdown reason reached_max_restart_intensity pid <5901.6043.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.6045.0> 2011-07-14 00:22:49 =============================================================================== ns_1@10.88.165.188:<5901.6045.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:49 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 265 times in the past 256.998755 secs (last seen 1.00011 secs ago INFO REPORT <5901.5830.0> 2011-07-14 00:22:49 =============================================================================== ns_1@10.88.165.188:<5901.5830.0>:single_bucket_sup:24: per-bucket supervisor for "default" died with reason shutdown SUPERVISOR REPORT <5901.273.0> 2011-07-14 00:22:49 =============================================================================== Reporting supervisor {local,ns_bucket_sup} Child process errorContext child_terminated reason shutdown pid <5901.5830.0> name {per_bucket_sup,"default"} mfargs {single_bucket_sup,start_link,["default"]} restart_type permanent shutdown infinity child_type supervisor PROGRESS REPORT <5901.273.0> 2011-07-14 00:22:49 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<5901.6055.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6057.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6058.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6059.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6060.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6061.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:49 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6062.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.6057.0> 2011-07-14 00:22:50 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.6057.0> 2011-07-14 00:22:50 =============================================================================== ** Generic server <5901.6057.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.53681>}, init, {1310,617369,902151}, "default",#Port<5901.3144>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.6057.0> 2011-07-14 00:22:50 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.6057.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.6055.0>] messages [] links [<5901.49.0>,<5901.6056.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.6056.0> 2011-07-14 00:22:50 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.6057.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:50 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 259 times in the past 257.658779 secs (last seen 1.003066 secs ago PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6067.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:50 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:50 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6068.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6069.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6070.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6071.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6072.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.6067.0> 2011-07-14 00:22:50 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.6067.0> 2011-07-14 00:22:50 =============================================================================== ** Generic server <5901.6067.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.53811>}, init, {1310,617370,405402}, "default",#Port<5901.3146>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.6067.0> 2011-07-14 00:22:50 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.6067.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.6055.0>] messages [] links [<5901.49.0>,<5901.6056.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.6056.0> 2011-07-14 00:22:50 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.6067.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.6069.0> 2011-07-14 00:22:50 =============================================================================== ns_1@10.88.165.188:<5901.6069.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:50 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 266 times in the past 257.99909 secs (last seen 1.000335 secs ago PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6076.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6077.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6078.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6079.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6080.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:50 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6081.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.6076.0> 2011-07-14 00:22:51 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.6076.0> 2011-07-14 00:22:51 =============================================================================== ** Generic server <5901.6076.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.53893>}, init, {1310,617370,902938}, "default",#Port<5901.3147>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.6076.0> 2011-07-14 00:22:51 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.6076.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.6055.0>] messages [] links [<5901.49.0>,<5901.6056.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.6056.0> 2011-07-14 00:22:51 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.6076.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:51 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 260 times in the past 258.658685 secs (last seen 0.999906 secs ago PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6086.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:51 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:51 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6087.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6088.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6089.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6090.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6091.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.6086.0> 2011-07-14 00:22:51 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.6086.0> 2011-07-14 00:22:51 =============================================================================== ** Generic server <5901.6086.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.53996>}, init, {1310,617371,405238}, "default",#Port<5901.3148>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.6086.0> 2011-07-14 00:22:51 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.6086.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.6055.0>] messages [] links [<5901.49.0>,<5901.6056.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.6056.0> 2011-07-14 00:22:51 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.6086.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.6088.0> 2011-07-14 00:22:51 =============================================================================== ns_1@10.88.165.188:<5901.6088.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:51 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 267 times in the past 258.998896 secs (last seen 0.999806 secs ago PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6098.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6099.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6100.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6101.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6102.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:51 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6103.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.6098.0> 2011-07-14 00:22:52 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.6098.0> 2011-07-14 00:22:52 =============================================================================== ** Generic server <5901.6098.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.54081>}, init, {1310,617371,902004}, "default",#Port<5901.3149>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.6098.0> 2011-07-14 00:22:52 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.6098.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.6055.0>] messages [] links [<5901.49.0>,<5901.6056.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.6056.0> 2011-07-14 00:22:52 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.6098.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:52 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 261 times in the past 259.658857 secs (last seen 1.000172 secs ago PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6107.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6108.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6109.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6110.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6111.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6112.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.6107.0> 2011-07-14 00:22:52 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.6107.0> 2011-07-14 00:22:52 =============================================================================== ** Generic server <5901.6107.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.54169>}, init, {1310,617372,405474}, "default",#Port<5901.3150>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.6107.0> 2011-07-14 00:22:52 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.6107.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.6055.0>] messages [] links [<5901.49.0>,<5901.6056.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.6056.0> 2011-07-14 00:22:52 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.6107.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.6109.0> 2011-07-14 00:22:52 =============================================================================== ns_1@10.88.165.188:<5901.6109.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:52 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 268 times in the past 259.999079 secs (last seen 1.000183 secs ago PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6116.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:52 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:52 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 6 log lines from memcached PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6117.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6118.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6119.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6120.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:52 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6121.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.6116.0> 2011-07-14 00:22:53 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.6116.0> 2011-07-14 00:22:53 =============================================================================== ** Generic server <5901.6116.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.54245>}, init, {1310,617372,903004}, "default",#Port<5901.3151>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.6116.0> 2011-07-14 00:22:53 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.6116.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.6055.0>] messages [] links [<5901.49.0>,<5901.6056.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.6056.0> 2011-07-14 00:22:53 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.6116.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:53 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 262 times in the past 260.659165 secs (last seen 1.000308 secs ago PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6126.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6127.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6128.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6129.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6130.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6131.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.6126.0> 2011-07-14 00:22:53 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.6126.0> 2011-07-14 00:22:53 =============================================================================== ** Generic server <5901.6126.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.54370>}, init, {1310,617373,405836}, "default",#Port<5901.3153>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.6126.0> 2011-07-14 00:22:53 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.6126.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.6055.0>] messages [] links [<5901.49.0>,<5901.6056.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6671 SUPERVISOR REPORT <5901.6056.0> 2011-07-14 00:22:53 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.6126.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.6128.0> 2011-07-14 00:22:53 =============================================================================== ns_1@10.88.165.188:<5901.6128.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:53 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 269 times in the past 260.997498 secs (last seen 0.998419 secs ago PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6144.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6145.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6147.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6148.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6149.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6150.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:53 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:53 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.6144.0> 2011-07-14 00:22:53 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.6144.0> 2011-07-14 00:22:53 =============================================================================== ** Generic server <5901.6144.0> terminating ** Last message in was list_vbuckets_prevstate ** When Server state == {state,{interval,#Ref<5901.0.0.54480>}, init, {1310,617373,900658}, "default",#Port<5901.3155>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.6144.0> 2011-07-14 00:22:53 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.6144.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.6055.0>] messages [] links [<5901.49.0>,<5901.6056.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6701 SUPERVISOR REPORT <5901.6056.0> 2011-07-14 00:22:53 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.6144.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker CRASH REPORT <5901.6138.0> 2011-07-14 00:22:53 =============================================================================== Crashing process initial_call {ns_janitor,cleanup,['Argument__1','Argument__2']} pid <5901.6138.0> registered_name [] error_info {exit,{{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, {gen_server,call, [{'ns_memcached-default','ns_1@10.88.165.188'}, list_vbuckets_prevstate,30000]}}, [{gen_server,call,3}, {ns_janitor,'-read_existing_map/4-lc$^0/1-0-',2}, {ns_janitor,read_existing_map,4}, {ns_janitor,cleanup,2}, {proc_lib,init_p_do_apply,3}]} ancestors [<5901.304.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<5901.52.0>] messages [] links [<5901.304.0>] dictionary [] trap_exit false status running heap_size 10946 stack_size 24 reductions 224 INFO REPORT <5901.227.0> 2011-07-14 00:22:53 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 270 times in the past 261.008315 secs (last seen 0.010817 secs ago INFO REPORT <5901.304.0> 2011-07-14 00:22:53 =============================================================================== ns_1@10.88.165.188:<5901.304.0>:ns_orchestrator:178: Janitor run exited for bucket "default" with reason {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, [{'ns_memcached-default', 'ns_1@10.88.165.188'}, list_vbuckets_prevstate, 30000]}} PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6152.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6153.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6154.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6155.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6156.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:53 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6157.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.6152.0> 2011-07-14 00:22:54 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.6152.0> 2011-07-14 00:22:54 =============================================================================== ** Generic server <5901.6152.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.54555>}, init, {1310,617373,910503}, "default",#Port<5901.3156>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.6152.0> 2011-07-14 00:22:54 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.6152.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.6055.0>] messages [] links [<5901.49.0>,<5901.6056.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.6056.0> 2011-07-14 00:22:54 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.6152.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:54 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 263 times in the past 261.666943 secs (last seen 1.007778 secs ago PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6162.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6163.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6164.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6165.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6166.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6167.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.6162.0> 2011-07-14 00:22:54 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.6162.0> 2011-07-14 00:22:54 =============================================================================== ** Generic server <5901.6162.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.54657>}, init, {1310,617374,413496}, "default",#Port<5901.3157>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.6162.0> 2011-07-14 00:22:54 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.6162.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.6055.0>] messages [] links [<5901.49.0>,<5901.6056.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.6056.0> 2011-07-14 00:22:54 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.6162.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.6164.0> 2011-07-14 00:22:54 =============================================================================== ns_1@10.88.165.188:<5901.6164.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:54 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 271 times in the past 261.999253 secs (last seen 0.990938 secs ago PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6171.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6172.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6173.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6174.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6175.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:54 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6176.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:54 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:54 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.6171.0> 2011-07-14 00:22:55 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.6171.0> 2011-07-14 00:22:55 =============================================================================== ** Generic server <5901.6171.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.54737>}, init, {1310,617374,902388}, "default",#Port<5901.3158>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.6171.0> 2011-07-14 00:22:55 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.6171.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.6055.0>] messages [] links [<5901.49.0>,<5901.6056.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.6056.0> 2011-07-14 00:22:55 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.6171.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:55 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 264 times in the past 262.656863 secs (last seen 0.98992 secs ago PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6180.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6181.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6182.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6183.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6184.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6185.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.6180.0> 2011-07-14 00:22:55 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.6180.0> 2011-07-14 00:22:55 =============================================================================== ** Generic server <5901.6180.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.54819>}, init, {1310,617375,407430}, "default",#Port<5901.3159>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.6180.0> 2011-07-14 00:22:55 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.6180.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.6055.0>] messages [] links [<5901.49.0>,<5901.6056.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.6056.0> 2011-07-14 00:22:55 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.6180.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.6182.0> 2011-07-14 00:22:55 =============================================================================== ns_1@10.88.165.188:<5901.6182.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:55 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 272 times in the past 262.999234 secs (last seen 0.999981 secs ago PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6192.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6193.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6194.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6195.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6196.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:55 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6197.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.6192.0> 2011-07-14 00:22:56 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.6192.0> 2011-07-14 00:22:56 =============================================================================== ** Generic server <5901.6192.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.54907>}, init, {1310,617375,902345}, "default",#Port<5901.3160>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.6192.0> 2011-07-14 00:22:56 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.6192.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.6055.0>] messages [] links [<5901.49.0>,<5901.6056.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.6056.0> 2011-07-14 00:22:56 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.6192.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:56 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 265 times in the past 263.656369 secs (last seen 0.999506 secs ago PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6202.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6203.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6204.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:56 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:56 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6205.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6206.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6207.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.231.0> 2011-07-14 00:22:56 =============================================================================== ns_log: logging ns_node_disco:5:Node 'ns_1@10.88.165.188' saw that node 'ns_1@10.88.23.62' went down. INFO REPORT <5901.230.0> 2011-07-14 00:22:56 =============================================================================== ns_node_disco_log: nodes changed: ['ns_1@10.88.165.188'] INFO REPORT <5901.6202.0> 2011-07-14 00:22:56 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.6202.0> 2011-07-14 00:22:56 =============================================================================== ** Generic server <5901.6202.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.55033>}, init, {1310,617376,406777}, "default",#Port<5901.3162>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.6202.0> 2011-07-14 00:22:56 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.6202.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.6055.0>] messages [] links [<5901.49.0>,<5901.6056.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.6056.0> 2011-07-14 00:22:56 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.6202.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.6204.0> 2011-07-14 00:22:56 =============================================================================== ns_1@10.88.165.188:<5901.6204.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:56 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 273 times in the past 263.999249 secs (last seen 1.000015 secs ago PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6211.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6212.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6213.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6214.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6215.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:56 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6216.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.6211.0> 2011-07-14 00:22:57 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.6211.0> 2011-07-14 00:22:57 =============================================================================== ** Generic server <5901.6211.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.55125>}, init, {1310,617376,907149}, "default",#Port<5901.3163>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.6211.0> 2011-07-14 00:22:57 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.6211.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.6055.0>] messages [] links [<5901.49.0>,<5901.6056.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.6056.0> 2011-07-14 00:22:57 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.6211.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:57 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 266 times in the past 264.659242 secs (last seen 1.002873 secs ago PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6219.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6220.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6221.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6222.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6223.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6224.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.267.0> 2011-07-14 00:22:57 =============================================================================== moxi<0.267.0>: 2011-07-14 00:22:58: (agent_config.c.705) ERROR: bad JSON configuration from http://127.0.0.1:8091/pools/default/saslBucketsStreaming: Number of buckets must be a power of two > 0 and <= MAX_BUCKETS ({ moxi<0.267.0>: "name": "default", moxi<0.267.0>: "nodeLocator": "vbucket", moxi<0.267.0>: "saslPassword": "", moxi<0.267.0>: "nodes": [{ moxi<0.267.0>: "replication": 1, moxi<0.267.0>: "clusterMembership": "active", moxi<0.267.0>: "status": "warmup", moxi<0.267.0>: "hostname": "10.88.165.188:8091", moxi<0.267.0>: "clusterCompatibility": 1, ERROR REPORT <5901.267.0> 2011-07-14 00:22:57 =============================================================================== ns_1@10.88.165.188:<5901.267.0>:ns_port_server:161: Dropped 24 log lines from moxi INFO REPORT <5901.6219.0> 2011-07-14 00:22:57 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.6219.0> 2011-07-14 00:22:57 =============================================================================== ** Generic server <5901.6219.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.55213>}, init, {1310,617377,405882}, "default",#Port<5901.3164>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.6219.0> 2011-07-14 00:22:57 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.6219.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.6055.0>] messages [] links [<5901.49.0>,<5901.6056.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.6056.0> 2011-07-14 00:22:57 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.6219.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.6221.0> 2011-07-14 00:22:57 =============================================================================== ns_1@10.88.165.188:<5901.6221.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:57 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 274 times in the past 264.999254 secs (last seen 1.000005 secs ago PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6230.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6231.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6232.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6233.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6234.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:57 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6235.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:57 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:57 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.6230.0> 2011-07-14 00:22:58 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.6230.0> 2011-07-14 00:22:58 =============================================================================== ** Generic server <5901.6230.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.55294>}, init, {1310,617377,902305}, "default",#Port<5901.3165>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.6230.0> 2011-07-14 00:22:58 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.6230.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.6055.0>] messages [] links [<5901.49.0>,<5901.6056.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.6056.0> 2011-07-14 00:22:58 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.6230.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:58 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 267 times in the past 265.656463 secs (last seen 0.997221 secs ago PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6238.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6239.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6240.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6241.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6242.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6243.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.6238.0> 2011-07-14 00:22:58 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.6238.0> 2011-07-14 00:22:58 =============================================================================== ** Generic server <5901.6238.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.55381>}, init, {1310,617378,407093}, "default",#Port<5901.3166>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.6238.0> 2011-07-14 00:22:58 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.6238.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.6055.0>] messages [] links [<5901.49.0>,<5901.6056.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6665 SUPERVISOR REPORT <5901.6056.0> 2011-07-14 00:22:58 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.6238.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.6240.0> 2011-07-14 00:22:58 =============================================================================== ns_1@10.88.165.188:<5901.6240.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:58 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 275 times in the past 265.999265 secs (last seen 1.000011 secs ago PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6249.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6251.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6252.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6253.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6254.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:58 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6255.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.6249.0> 2011-07-14 00:22:59 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.6249.0> 2011-07-14 00:22:59 =============================================================================== ** Generic server <5901.6249.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.55556>}, init, {1310,617378,902395}, "default",#Port<5901.3168>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.6249.0> 2011-07-14 00:22:59 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.6249.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.6055.0>] messages [] links [<5901.49.0>,<5901.6056.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7252 SUPERVISOR REPORT <5901.6056.0> 2011-07-14 00:22:59 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.6249.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:22:59 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 268 times in the past 266.656481 secs (last seen 1.000018 secs ago PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6260.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6261.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6262.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6263.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6264.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6056.0> 2011-07-14 00:22:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6265.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:22:59 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:22:59 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.6260.0> 2011-07-14 00:22:59 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.6260.0> 2011-07-14 00:22:59 =============================================================================== ** Generic server <5901.6260.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.55682>}, init, {1310,617379,403034}, "default",#Port<5901.3170>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.6260.0> 2011-07-14 00:22:59 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.6260.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.6055.0>] messages [] links [<5901.49.0>,<5901.6056.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.6056.0> 2011-07-14 00:22:59 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.6260.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker SUPERVISOR REPORT <5901.6056.0> 2011-07-14 00:22:59 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext shutdown reason reached_max_restart_intensity pid <5901.6260.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.6262.0> 2011-07-14 00:22:59 =============================================================================== ns_1@10.88.165.188:<5901.6262.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:22:59 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 276 times in the past 266.999429 secs (last seen 1.000164 secs ago INFO REPORT <5901.6055.0> 2011-07-14 00:22:59 =============================================================================== ns_1@10.88.165.188:<5901.6055.0>:single_bucket_sup:24: per-bucket supervisor for "default" died with reason shutdown SUPERVISOR REPORT <5901.273.0> 2011-07-14 00:22:59 =============================================================================== Reporting supervisor {local,ns_bucket_sup} Child process errorContext child_terminated reason shutdown pid <5901.6055.0> name {per_bucket_sup,"default"} mfargs {single_bucket_sup,start_link,["default"]} restart_type permanent shutdown infinity child_type supervisor PROGRESS REPORT <5901.273.0> 2011-07-14 00:22:59 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<5901.6271.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <5901.6272.0> 2011-07-14 00:22:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6273.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.6272.0> 2011-07-14 00:22:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6274.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6272.0> 2011-07-14 00:22:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6275.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6272.0> 2011-07-14 00:22:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6276.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6272.0> 2011-07-14 00:22:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6277.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6272.0> 2011-07-14 00:22:59 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6278.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.6273.0> 2011-07-14 00:23:00 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.6273.0> 2011-07-14 00:23:00 =============================================================================== ** Generic server <5901.6273.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.55767>}, init, {1310,617379,905340}, "default",#Port<5901.3171>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.6273.0> 2011-07-14 00:23:00 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.6273.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.6271.0>] messages [] links [<5901.49.0>,<5901.6272.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.6272.0> 2011-07-14 00:23:00 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.6273.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:23:00 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 269 times in the past 267.656772 secs (last seen 1.000291 secs ago PROGRESS REPORT <5901.6272.0> 2011-07-14 00:23:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6281.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.6272.0> 2011-07-14 00:23:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6282.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6272.0> 2011-07-14 00:23:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6283.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6272.0> 2011-07-14 00:23:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6284.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6272.0> 2011-07-14 00:23:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6285.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6272.0> 2011-07-14 00:23:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6286.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.6281.0> 2011-07-14 00:23:00 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.6281.0> 2011-07-14 00:23:00 =============================================================================== ** Generic server <5901.6281.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.55856>}, init, {1310,617380,403285}, "default",#Port<5901.3172>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.6281.0> 2011-07-14 00:23:00 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.6281.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.6271.0>] messages [] links [<5901.49.0>,<5901.6272.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.6272.0> 2011-07-14 00:23:00 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.6281.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.6283.0> 2011-07-14 00:23:00 =============================================================================== ns_1@10.88.165.188:<5901.6283.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:23:00 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 277 times in the past 267.999643 secs (last seen 1.000214 secs ago PROGRESS REPORT <5901.6272.0> 2011-07-14 00:23:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6289.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.6272.0> 2011-07-14 00:23:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6290.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6272.0> 2011-07-14 00:23:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6291.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6272.0> 2011-07-14 00:23:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6292.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6272.0> 2011-07-14 00:23:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6293.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6272.0> 2011-07-14 00:23:00 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6294.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.269.0> 2011-07-14 00:23:00 =============================================================================== memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: Failed to create database: unable to open database file while building query: ``-- Initialization parameters for sqlite backend for membase buckets. memcached<0.269.0>: pragma auto_vacuum = none;'' memcached<0.269.0>: No access to "/opt/membase/var/lib/membase/data/default-data/default". memcached<0.269.0>: memcached<0.269.0>: sqlite error: unable to open database file memcached<0.269.0>: sqlite error: unable to open database file ERROR REPORT <5901.269.0> 2011-07-14 00:23:00 =============================================================================== ns_1@10.88.165.188:<5901.269.0>:ns_port_server:161: Dropped 14 log lines from memcached INFO REPORT <5901.6289.0> 2011-07-14 00:23:01 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, has_started, 1}, {ns_memcached, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.6289.0> 2011-07-14 00:23:01 =============================================================================== ** Generic server <5901.6289.0> terminating ** Last message in was check_started ** When Server state == {state,{interval,#Ref<5901.0.0.55932>}, init, {1310,617380,902654}, "default",#Port<5901.3173>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.6289.0> 2011-07-14 00:23:01 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.6289.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.6271.0>] messages [] links [<5901.49.0>,<5901.6272.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 7204 SUPERVISOR REPORT <5901.6272.0> 2011-07-14 00:23:01 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,has_started,1}, {ns_memcached,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.6289.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker INFO REPORT <5901.227.0> 2011-07-14 00:23:01 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n has_started,\n 1},\n {ns_memcached,\n handle_info,\n 2},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 270 times in the past 268.656771 secs (last seen 0.999999 secs ago PROGRESS REPORT <5901.6272.0> 2011-07-14 00:23:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6297.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.6272.0> 2011-07-14 00:23:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6298.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6272.0> 2011-07-14 00:23:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6299.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6272.0> 2011-07-14 00:23:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6300.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6272.0> 2011-07-14 00:23:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6301.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6272.0> 2011-07-14 00:23:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6302.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] INFO REPORT <5901.6297.0> 2011-07-14 00:23:01 =============================================================================== ns_log: logging ns_memcached:4:Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} ERROR REPORT <5901.6297.0> 2011-07-14 00:23:01 =============================================================================== ** Generic server <5901.6297.0> terminating ** Last message in was {stats,<<>>} ** When Server state == {state,{interval,#Ref<5901.0.0.56018>}, init, {1310,617381,403443}, "default",#Port<5901.3174>} ** Reason for termination == ** {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} CRASH REPORT <5901.6297.0> 2011-07-14 00:23:01 =============================================================================== Crashing process initial_call {ns_memcached,init,['Argument__1']} pid <5901.6297.0> registered_name [] error_info {exit,{{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors ['single_bucket_sup-default',<5901.6271.0>] messages [] links [<5901.49.0>,<5901.6272.0>] dictionary [] trap_exit true status running heap_size 10946 stack_size 24 reductions 6659 SUPERVISOR REPORT <5901.6272.0> 2011-07-14 00:23:01 =============================================================================== Reporting supervisor {local,'single_bucket_sup-default'} Child process errorContext child_terminated reason {{badmatch,{error,closed}}, [{mc_client_binary,stats_recv,4}, {mc_client_binary,stats,4}, {ns_memcached,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} pid <5901.6297.0> name {ns_memcached,"default"} mfargs {ns_memcached,start_link,["default"]} restart_type permanent shutdown 86400000 child_type worker ERROR REPORT <5901.6299.0> 2011-07-14 00:23:01 =============================================================================== ns_1@10.88.165.188:<5901.6299.0>:stats_collector:94: Exception in stats collector: {exit, {{{badmatch, {error, closed}}, [{mc_client_binary, stats_recv, 4}, {mc_client_binary, stats, 4}, {ns_memcached, handle_call, 3}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]}, {gen_server, call, ['ns_memcached-default', {stats, <<>>}, 30000]}}, [{gen_server, call, 3}, {stats_collector, grab_all_stats, 1}, {stats_collector, handle_info, 2}, {gen_server, handle_msg, 5}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <5901.227.0> 2011-07-14 00:23:01 =============================================================================== ns_log: suppressing duplicate log ns_memcached:4("Control connection to memcached on 'ns_1@10.88.165.188' disconnected: {{badmatch,\n {error,\n closed}},\n [{mc_client_binary,\n stats_recv,\n 4},\n {mc_client_binary,\n stats,\n 4},\n {ns_memcached,\n handle_call,\n 3},\n {gen_server,\n handle_msg,\n 5},\n {proc_lib,\n init_p_do_apply,\n 3}]}") because it's been seen 278 times in the past 268.999707 secs (last seen 1.000064 secs ago PROGRESS REPORT <5901.6272.0> 2011-07-14 00:23:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6308.0>}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <5901.6272.0> 2011-07-14 00:23:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6309.0>}, {name,{ns_vbm_sup,"default"}}, {mfargs,{ns_vbm_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6272.0> 2011-07-14 00:23:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6310.0>}, {name,{stats_collector,"default"}}, {mfargs,{stats_collector,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6272.0> 2011-07-14 00:23:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6311.0>}, {name,{stats_archiver,"default"}}, {mfargs,{stats_archiver,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6272.0> 2011-07-14 00:23:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6312.0>}, {name,{stats_reader,"default"}}, {mfargs,{stats_reader,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <5901.6272.0> 2011-07-14 00:23:01 =============================================================================== supervisor {local,'single_bucket_sup-default'} started [{pid,<5901.6313.0>}, {name,{failover_safeness_level,"default"}}, {mfargs,{failover_safeness_level,start_link,["default"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}]