memcached stuck when warming up at (bucket_engine.c:253 waiting for mutex lock ? )

Last login: Thu Apr 26 11:21:02 on ttys020
farshid-2:~ farshid$ ssh farshid@10.3.121.148
farshid@10.3.121.148's password:
^C
farshid-2:~ farshid$ ssh farshid@10.3.121.148 -i farshid-key.pem
farshid@10.3.121.148's password:

farshid-2:~ farshid$ ssh root@10.3.121.148
root@10.3.121.148's password:
Permission denied, please try again.
root@10.3.121.148's password:
Last login: Fri Apr 20 12:09:06 2012 from 10.2.2.166
[root@centos-57_vm001 ~]# ls
cbsetup.sh hw ip vm.json
[root@centos-57_vm001 ~]# ls
cbsetup.sh hw ip vm.json
[root@centos-57_vm001 ~]# cd /tmp/
[root@centos-57_vm001 tmp]# ls
backup core.memcached.22252 core.memcached.9330
core.memcached.11581 core.memcached.22321 couchbase-server-enterprise_x86_64_2.0.0r-1149-rel.rpm
[root@centos-57_vm001 tmp]# ls -l
total 2343448
drwxr-xr-x 2 root root 4096 Apr 19 17:19 backup
rw------ 1 couchbase couchbase 826486784 Apr 20 12:34 core.memcached.11581
rw------ 1 couchbase couchbase 453308416 Apr 23 12:38 core.memcached.22252
rw------ 1 couchbase couchbase 789745664 Apr 23 13:45 core.memcached.22321
rw------ 1 couchbase couchbase 837054464 Apr 19 17:48 core.memcached.9330
rw-rr- 1 root root 81237855 Apr 25 13:08 couchbase-server-enterprise_x86_64_2.0.0r-1149-rel.rpm
[root@centos-57_vm001 tmp]# date
Thu Apr 26 09:25:52 EDT 2012
[root@centos-57_vm001 tmp]# rm -rf core.memcached.*
[root@centos-57_vm001 tmp]# top

top - 09:30:10 up 7 days, 19:34, 1 user, load average: 0.00, 0.00, 0.00
Tasks: 53 total, 1 running, 52 sleeping, 0 stopped, 0 zombie
Cpu(s): 4.7%us, 1.7%sy, 0.0%ni, 92.7%id, 0.5%wa, 0.0%hi, 0.2%si, 0.2%st
Mem: 4194304k total, 2191528k used, 2002776k free, 496016k buffers
Swap: 0k total, 0k used, 0k free, 1294128k cached

PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
1 root 15 0 10368 772 652 S 0.0 0.0 0:00.08 init
2 root RT -5 0 0 0 S 0.0 0.0 0:00.02 migration/0
3 root 34 19 0 0 0 S 0.0 0.0 0:00.01 ksoftirqd/0
4 root RT -5 0 0 0 S 0.0 0.0 0:00.00 watchdog/0
5 root 10 -5 0 0 0 S 0.0 0.0 0:00.01 events/0
6 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 khelper
7 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 kthread
9 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 xenwatch
10 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 xenbus
14 root RT -5 0 0 0 S 0.0 0.0 0:00.04 migration/1
15 root 34 19 0 0 0 S 0.0 0.0 0:00.02 ksoftirqd/1
16 root RT -5 0 0 0 S 0.0 0.0 0:00.00 watchdog/1
17 root 10 -5 0 0 0 S 0.0 0.0 0:00.01 events/1
20 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 kblockd/0
21 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 kblockd/1
22 root 20 -5 0 0 0 S 0.0 0.0 0:00.00 cqueue/0
23 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 cqueue/1
[root@centos-57_vm001 tmp]# top

top - 09:30:18 up 7 days, 19:34, 1 user, load average: 0.00, 0.00, 0.00
Tasks: 53 total, 1 running, 52 sleeping, 0 stopped, 0 zombie
Cpu(s): 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st
Mem: 4194304k total, 2191148k used, 2003156k free, 496016k buffers
Swap: 0k total, 0k used, 0k free, 1294148k cached

PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
1 root 15 0 10368 772 652 S 0.0 0.0 0:00.08 init
2 root RT -5 0 0 0 S 0.0 0.0 0:00.02 migration/0
3 root 34 19 0 0 0 S 0.0 0.0 0:00.01 ksoftirqd/0
4 root RT -5 0 0 0 S 0.0 0.0 0:00.00 watchdog/0
5 root 10 -5 0 0 0 S 0.0 0.0 0:00.01 events/0
6 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 khelper
7 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 kthread
9 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 xenwatch
10 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 xenbus
14 root RT -5 0 0 0 S 0.0 0.0 0:00.04 migration/1
15 root 34 19 0 0 0 S 0.0 0.0 0:00.02 ksoftirqd/1
16 root RT -5 0 0 0 S 0.0 0.0 0:00.00 watchdog/1
17 root 10 -5 0 0 0 S 0.0 0.0 0:00.01 events/1
20 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 kblockd/0
21 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 kblockd/1
22 root 20 -5 0 0 0 S 0.0 0.0 0:00.00 cqueue/0
23 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 cqueue/1
[root@centos-57_vm001 tmp]#
[root@centos-57_vm001 tmp]# ifconfig
eth0 Link encap:Ethernet HWaddr 1E:26:36:99:6D:E6
inet addr:10.3.121.148 Bcast:10.3.255.255 Mask:255.255.0.0
inet6 addr: fe80::1c26:36ff:fe99:6de6/64 Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:141881091 errors:0 dropped:0 overruns:0 frame:0
TX packets:137791675 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:92537705061 (86.1 GiB) TX bytes:102256393406 (95.2 GiB)

lo Link encap:Local Loopback
inet addr:127.0.0.1 Mask:255.0.0.0
inet6 addr: ::1/128 Scope:Host
UP LOOPBACK RUNNING MTU:16436 Metric:1
RX packets:56037752 errors:0 dropped:0 overruns:0 frame:0
TX packets:56037752 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:75433906748 (70.2 GiB) TX bytes:75433906748 (70.2 GiB)

[root@centos-57_vm001 tmp]#
[root@centos-57_vm001 tmp]# top

top - 13:30:12 up 7 days, 23:34, 1 user, load average: 0.00, 0.00, 0.00
Tasks: 53 total, 1 running, 52 sleeping, 0 stopped, 0 zombie
Cpu(s): 4.6%us, 1.7%sy, 0.0%ni, 92.9%id, 0.5%wa, 0.0%hi, 0.2%si, 0.2%st
Mem: 4194304k total, 2234180k used, 1960124k free, 496060k buffers
Swap: 0k total, 0k used, 0k free, 1335992k cached

PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
1 root 15 0 10368 772 652 S 0.0 0.0 0:00.08 init
2 root RT -5 0 0 0 S 0.0 0.0 0:00.02 migration/0
3 root 34 19 0 0 0 S 0.0 0.0 0:00.01 ksoftirqd/0
4 root RT -5 0 0 0 S 0.0 0.0 0:00.00 watchdog/0
5 root 10 -5 0 0 0 S 0.0 0.0 0:00.01 events/0
6 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 khelper
7 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 kthread
9 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 xenwatch
10 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 xenbus
14 root RT -5 0 0 0 S 0.0 0.0 0:00.04 migration/1
15 root 34 19 0 0 0 S 0.0 0.0 0:00.02 ksoftirqd/1
16 root RT -5 0 0 0 S 0.0 0.0 0:00.00 watchdog/1
17 root 10 -5 0 0 0 S 0.0 0.0 0:00.01 events/1
20 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 kblockd/0
21 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 kblockd/1
22 root 20 -5 0 0 0 S 0.0 0.0 0:00.00 cqueue/0
23 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 cqueue/1
[root@centos-57_vm001 tmp]# to
[root@centos-57_vm001 tmp]# /etc/init.d/couchbase-server start
couchbase-server is already started [WARNING]
[root@centos-57_vm001 tmp]#
[root@centos-57_vm001 tmp]# ps -aef
UID PID PPID C STIME TTY TIME CMD
root 1 0 0 Apr18 ? 00:00:00 init [3]
root 2 1 0 Apr18 ? 00:00:00 [migration/0]
root 3 1 0 Apr18 ? 00:00:00 [ksoftirqd/0]
root 4 1 0 Apr18 ? 00:00:00 [watchdog/0]
root 5 1 0 Apr18 ? 00:00:00 [events/0]
root 6 1 0 Apr18 ? 00:00:00 [khelper]
root 7 1 0 Apr18 ? 00:00:00 [kthread]
root 9 7 0 Apr18 ? 00:00:00 [xenwatch]
root 10 7 0 Apr18 ? 00:00:00 [xenbus]
root 14 7 0 Apr18 ? 00:00:00 [migration/1]
root 15 7 0 Apr18 ? 00:00:00 [ksoftirqd/1]
root 16 7 0 Apr18 ? 00:00:00 [watchdog/1]
root 17 7 0 Apr18 ? 00:00:00 [events/1]
root 20 7 0 Apr18 ? 00:00:00 [kblockd/0]
root 21 7 0 Apr18 ? 00:00:00 [kblockd/1]
root 22 7 0 Apr18 ? 00:00:00 [cqueue/0]
root 23 7 0 Apr18 ? 00:00:00 [cqueue/1]
root 27 7 0 Apr18 ? 00:00:00 [khubd]
root 29 7 0 Apr18 ? 00:00:00 [kseriod]
root 98 7 0 Apr18 ? 00:00:00 [khungtaskd]
root 101 7 0 Apr18 ? 00:00:02 [kswapd0]
root 102 7 0 Apr18 ? 00:00:00 [aio/0]
root 103 7 0 Apr18 ? 00:00:00 [aio/1]
root 233 7 0 Apr18 ? 00:00:00 [kpsmoused]
root 261 7 0 Apr18 ? 00:00:00 [kstriped]
root 277 7 0 Apr18 ? 00:07:26 [kjournald]
root 299 7 0 Apr18 ? 00:00:00 [kauditd]
root 327 1 0 Apr18 ? 00:00:00 /sbin/udevd -d
root 633 7 0 Apr18 ? 00:00:00 [kmpathd/0]
root 634 7 0 Apr18 ? 00:00:00 [kmpathd/1]
root 635 7 0 Apr18 ? 00:00:00 [kmpath_handlerd]
root 870 1 0 Apr18 ? 00:00:00 /sbin/dhclient 1 -q -lf /var/lib/dhclient/dhclient
root 950 1 0 Apr18 ? 00:00:00 /sbin/rsyslogd -m 0
root 1019 1 0 Apr18 ? 00:00:00 /usr/sbin/sshd
root 1027 1 0 Apr18 ? 00:00:00 xinetd -stayalive -pidfile /var/run/xinetd.pid
root 1035 1 0 Apr18 ? 00:00:00 crond
root 1041 1 0 Apr18 xvc0 00:00:00 /sbin/agetty xvc0 9600 vt100-nav
root 9420 7 0 Apr19 ? 00:00:00 [pdflush]
root 22444 7 0 Apr23 ? 00:00:00 [pdflush]
100 28277 1 0 Apr25 ? 00:00:00 /opt/couchbase/lib/erlang/erts-5.8.5/bin/epmd -daemo
100 28290 1 9 Apr25 ? 01:39:01 /opt/couchbase/lib/erlang/erts-5.8.5/bin/beam.smp -A
100 28351 28290 0 Apr25 ? 00:00:00 sh -s disksup
100 28352 28290 0 Apr25 ? 00:00:00 /opt/couchbase/lib/erlang/lib/os_mon-2.2.7/priv/bin/
100 28354 28290 0 Apr25 ? 00:00:00 /opt/couchbase/lib/erlang/lib/os_mon-2.2.7/priv/bin/
100 28355 28290 0 Apr25 ? 00:00:00 /opt/couchbase/lib/erlang/lib/ssl-4.1.6/priv/bin/ssl
100 28407 28290 0 Apr25 ? 00:00:00 inet_gethost 4
100 28408 28407 0 Apr25 ? 00:00:00 inet_gethost 4
100 28427 28407 0 Apr25 ? 00:00:00 inet_gethost 4
100 28429 28407 0 Apr25 ? 00:00:00 inet_gethost 4
100 28519 28407 0 Apr25 ? 00:00:00 inet_gethost 4
root 29667 1019 0 09:25 ? 00:00:00 sshd: root@pts/0
root 29669 29667 0 09:25 pts/0 00:00:00 -bash
root 29990 29669 0 13:30 pts/0 00:00:00 ps -aef
[root@centos-57_vm001 tmp]# ps -aef
UID PID PPID C STIME TTY TIME CMD
root 1 0 0 Apr18 ? 00:00:00 init [3]
root 2 1 0 Apr18 ? 00:00:00 [migration/0]
root 3 1 0 Apr18 ? 00:00:00 [ksoftirqd/0]
root 4 1 0 Apr18 ? 00:00:00 [watchdog/0]
root 5 1 0 Apr18 ? 00:00:00 [events/0]
root 6 1 0 Apr18 ? 00:00:00 [khelper]
root 7 1 0 Apr18 ? 00:00:00 [kthread]
root 9 7 0 Apr18 ? 00:00:00 [xenwatch]
root 10 7 0 Apr18 ? 00:00:00 [xenbus]
root 14 7 0 Apr18 ? 00:00:00 [migration/1]
root 15 7 0 Apr18 ? 00:00:00 [ksoftirqd/1]
root 16 7 0 Apr18 ? 00:00:00 [watchdog/1]
root 17 7 0 Apr18 ? 00:00:00 [events/1]
root 20 7 0 Apr18 ? 00:00:00 [kblockd/0]
root 21 7 0 Apr18 ? 00:00:00 [kblockd/1]
root 22 7 0 Apr18 ? 00:00:00 [cqueue/0]
root 23 7 0 Apr18 ? 00:00:00 [cqueue/1]
root 27 7 0 Apr18 ? 00:00:00 [khubd]
root 29 7 0 Apr18 ? 00:00:00 [kseriod]
root 98 7 0 Apr18 ? 00:00:00 [khungtaskd]
root 101 7 0 Apr18 ? 00:00:02 [kswapd0]
root 102 7 0 Apr18 ? 00:00:00 [aio/0]
root 103 7 0 Apr18 ? 00:00:00 [aio/1]
root 233 7 0 Apr18 ? 00:00:00 [kpsmoused]
root 261 7 0 Apr18 ? 00:00:00 [kstriped]
root 277 7 0 Apr18 ? 00:07:26 [kjournald]
root 299 7 0 Apr18 ? 00:00:00 [kauditd]
root 327 1 0 Apr18 ? 00:00:00 /sbin/udevd -d
root 633 7 0 Apr18 ? 00:00:00 [kmpathd/0]
root 634 7 0 Apr18 ? 00:00:00 [kmpathd/1]
root 635 7 0 Apr18 ? 00:00:00 [kmpath_handlerd]
root 870 1 0 Apr18 ? 00:00:00 /sbin/dhclient -1 -q -lf /var/lib/dhclient/dhclient-eth0.leases -pf /var/run/dhclient-e
root 950 1 0 Apr18 ? 00:00:00 /sbin/rsyslogd -m 0
root 1019 1 0 Apr18 ? 00:00:00 /usr/sbin/sshd
root 1027 1 0 Apr18 ? 00:00:00 xinetd -stayalive -pidfile /var/run/xinetd.pid
root 1035 1 0 Apr18 ? 00:00:00 crond
root 1041 1 0 Apr18 xvc0 00:00:00 /sbin/agetty xvc0 9600 vt100-nav
root 9420 7 0 Apr19 ? 00:00:00 [pdflush]
root 22444 7 0 Apr23 ? 00:00:00 [pdflush]
100 28277 1 0 Apr25 ? 00:00:00 /opt/couchbase/lib/erlang/erts-5.8.5/bin/epmd -daemon
100 28290 1 9 Apr25 ? 01:39:01 /opt/couchbase/lib/erlang/erts-5.8.5/bin/beam.smp -A 16 -sbt u -P 327680 -K true – -ro
100 28351 28290 0 Apr25 ? 00:00:00 sh -s disksup
100 28352 28290 0 Apr25 ? 00:00:00 /opt/couchbase/lib/erlang/lib/os_mon-2.2.7/priv/bin/memsup
100 28354 28290 0 Apr25 ? 00:00:00 /opt/couchbase/lib/erlang/lib/os_mon-2.2.7/priv/bin/cpu_sup
100 28355 28290 0 Apr25 ? 00:00:00 /opt/couchbase/lib/erlang/lib/ssl-4.1.6/priv/bin/ssl_esock
100 28407 28290 0 Apr25 ? 00:00:00 inet_gethost 4
100 28408 28407 0 Apr25 ? 00:00:00 inet_gethost 4
100 28427 28407 0 Apr25 ? 00:00:00 inet_gethost 4
100 28429 28407 0 Apr25 ? 00:00:00 inet_gethost 4
100 28519 28407 0 Apr25 ? 00:00:00 inet_gethost 4
root 29667 1019 0 09:25 ? 00:00:00 sshd: root@pts/0
root 29669 29667 0 09:25 pts/0 00:00:00 -bash
root 29991 29669 0 13:30 pts/0 00:00:00 ps -aef
[root@centos-57_vm001 tmp]#
[root@centos-57_vm001 tmp]# tail -f /opt/couchbase/var/lib/couchbase/logs/
couchdb.1 debug.10 debug.14 debug.2 debug.6 debug.idx errors.3 log.2 log.6 log.idx views.3
couchdb.idx debug.11 debug.15 debug.3 debug.7 debug.siz errors.idx log.3 log.7 log.siz views.4
couchdb.siz debug.12 debug.16 debug.4 debug.8 errors.1 errors.siz log.4 log.8 views.1 views.idx
debug.1 debug.13 debug.17 debug.5 debug.9 errors.2 log.1 log.5 log.9 views.2 views.siz
[root@centos-57_vm001 tmp]# tail -f /opt/couchbase/var/lib/couchbase/logs/errors.3
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [

{pid,<0.21742.35>}

,

{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]


[error_logger:error] [2012-04-26 13:30:59] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.3.121.148'},
{latest,minute,1}]}}
Offender: [{pid,<0.21765.35>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]


[error_logger:error] [2012-04-26 13:31:01] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'ns_memcached_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,stats,"default",0}},
{mfargs,{ns_memcached,start_link,[{"default",stats,0}]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]


[error_logger:error] [2012-04-26 13:31:01] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.21767.35>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['ns_memcached_sup-default','single_bucket_sup-default',
<0.30576.17>]
messages: [{'$gen_call',{<0.21795.35>,#Ref<0.0.240.21845>},topkeys}]
links: [<0.30579.17>]
dictionary: []
trap_exit: true
status: running
heap_size: 196418
stack_size: 24
reductions: 18335
neighbours:

[error_logger:error] [2012-04-26 13:31:01] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.21768.35>},
{name,hot_keys_keeper}

,
{mfargs,{hot_keys_keeper,start_link,[]}},

{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]


[error_logger:error] [2012-04-26 13:31:02] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.3.121.148'},
{latest,minute,1}]}}
Offender: [{pid,<0.21786.35>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent}

,

{shutdown,5000},
{child_type,worker}]


[error_logger:error] [2012-04-26 13:31:05] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.3.121.148'},
{latest,minute,1}]}}
Offender: [{pid,<0.21799.35>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000}

,

{child_type,worker}]


[error_logger:error] [2012-04-26 13:31:06] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'ns_memcached_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,stats,"default",0}},
{mfargs,{ns_memcached,start_link,[{"default",stats,0}]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}

]

[error_logger:error] [2012-04-26 13:31:06] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.21796.35>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['ns_memcached_sup-default','single_bucket_sup-default',
<0.30576.17>]
messages: [{'$gen_call',{<0.21819.35>,
{#Ref<0.0.240.22007>,'ns_1@10.3.121.148'}},
connected},
{'$gen_call',

{<0.21820.35>,#Ref<0.0.240.22045>}

,topkeys}]
links: [<0.30579.17>]
dictionary: []
trap_exit: true
status: running
heap_size: 196418
stack_size: 24
reductions: 18339
neighbours:

[error_logger:error] [2012-04-26 13:31:06] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor:

{local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.21797.35>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]


[error_logger:error] [2012-04-26 13:31:08] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}

Context: child_terminated
Reason: {noproc,
{gen_server,call,
[

{'stats_reader-default','ns_1@10.3.121.148'},
{latest,minute,1}]}}
Offender: [{pid,<0.21817.35>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]



[root@centos-57_vm001 tmp]# cd /tmp/
[root@centos-57_vm001 tmp]# ls
backup couchbase-server-enterprise_x86_64_2.0.0r-1149-rel.rpm
[root@centos-57_vm001 tmp]# /etc/init.d/couchbase-server restart
Stopping couchbase-server


BREAK: (a)bort (c)ontinue (p)roc info (i)nfo (l)oaded
(v)ersion (k)ill (D)b-tables (d)istribution
Killing not allowed - living nodes in database.

Starting couchbase-server
[1]+ Stopped /etc/init.d/couchbase-server restart
[root@centos-57_vm001 tmp]# killall -9 beam.smp
[root@centos-57_vm001 tmp]# killall -9 beam.smp
beam.smp: no process killed
[root@centos-57_vm001 tmp]# /etc/init.d/couchbase-server start
Starting couchbase-server [ OK ]
[root@centos-57_vm001 tmp]# top

top - 13:49:56 up 7 days, 23:54, 1 user, load average: 0.91, 0.68, 0.28
Tasks: 56 total, 2 running, 51 sleeping, 2 stopped, 1 zombie
Cpu(s): 0.0%us, 0.0%sy, 0.0%ni, 99.8%id, 0.0%wa, 0.0%hi, 0.0%si, 0.2%st
Mem: 4194304k total, 1990772k used, 2203532k free, 496784k buffers
Swap: 0k total, 0k used, 0k free, 1141052k cached

PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
1 root 15 0 10368 772 652 S 0.0 0.0 0:00.08 init
2 root RT -5 0 0 0 S 0.0 0.0 0:00.02 migration/0
3 root 34 19 0 0 0 S 0.0 0.0 0:00.01 ksoftirqd/0
4 root RT -5 0 0 0 S 0.0 0.0 0:00.00 watchdog/0
5 root 10 -5 0 0 0 S 0.0 0.0 0:00.01 events/0
6 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 khelper
7 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 kthread
9 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 xenwatch
10 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 xenbus
14 root RT -5 0 0 0 S 0.0 0.0 0:00.04 migration/1
15 root 34 19 0 0 0 S 0.0 0.0 0:00.02 ksoftirqd/1
16 root RT -5 0 0 0 S 0.0 0.0 0:00.00 watchdog/1
17 root 10 -5 0 0 0 S 0.0 0.0 0:00.01 events/1
20 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 kblockd/0
21 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 kblockd/1
22 root 20 -5 0 0 0 S 0.0 0.0 0:00.00 cqueue/0
23 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 cqueue/1
27 root 20 -5 0 0 0 S 0.0 0.0 0:00.00 khubd
29 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 kseriod
98 root 15 0 0 0 0 S 0.0 0.0 0:00.00 khungtaskd
101 root 10 -5 0 0 0 S 0.0 0.0 0:02.14 kswapd0
102 root 12 -5 0 0 0 S 0.0 0.0 0:00.00 aio/0
103 root 12 -5 0 0 0 S 0.0 0.0 0:00.00 aio/1
233 root 11 -5 0 0 0 S 0.0 0.0 0:00.00 kpsmoused
261 root 14 -5 0 0 0 S 0.0 0.0 0:00.00 kstriped
277 root 10 -5 0 0 0 S 0.0 0.0 7:26.83 kjournald
299 root 11 -5 0 0 0 S 0.0 0.0 0:00.00 kauditd
[root@centos-57_vm001 tmp]# top

top - 13:49:58 up 7 days, 23:54, 1 user, load average: 0.91, 0.68, 0.28
Tasks: 56 total, 1 running, 52 sleeping, 2 stopped, 1 zombie
Cpu(s): 4.6%us, 1.7%sy, 0.0%ni, 92.9%id, 0.5%wa, 0.0%hi, 0.2%si, 0.2%st
Mem: 4194304k total, 1990556k used, 2203748k free, 496784k buffers
Swap: 0k total, 0k used, 0k free, 1141116k cached

PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
1 root 15 0 10368 772 652 S 0.0 0.0 0:00.08 init
2 root RT -5 0 0 0 S 0.0 0.0 0:00.02 migration/0
3 root 34 19 0 0 0 S 0.0 0.0 0:00.01 ksoftirqd/0
4 root RT -5 0 0 0 S 0.0 0.0 0:00.00 watchdog/0
5 root 10 -5 0 0 0 S 0.0 0.0 0:00.01 events/0
6 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 khelper
7 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 kthread
9 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 xenwatch
10 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 xenbus
14 root RT -5 0 0 0 S 0.0 0.0 0:00.04 migration/1
15 root 34 19 0 0 0 S 0.0 0.0 0:00.02 ksoftirqd/1
16 root RT -5 0 0 0 S 0.0 0.0 0:00.00 watchdog/1
17 root 10 -5 0 0 0 S 0.0 0.0 0:00.01 events/1
20 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 kblockd/0
21 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 kblockd/1
22 root 20 -5 0 0 0 S 0.0 0.0 0:00.00 cqueue/0
23 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 cqueue/1
27 root 20 -5 0 0 0 S 0.0 0.0 0:00.00 khubd
29 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 kseriod
98 root 15 0 0 0 0 S 0.0 0.0 0:00.00 khungtaskd
101 root 10 -5 0 0 0 S 0.0 0.0 0:02.14 kswapd0
102 root 12 -5 0 0 0 S 0.0 0.0 0:00.00 aio/0
103 root 12 -5 0 0 0 S 0.0 0.0 0:00.00 aio/1
233 root 11 -5 0 0 0 S 0.0 0.0 0:00.00 kpsmoused
261 root 14 -5 0 0 0 S 0.0 0.0 0:00.00 kstriped
277 root 10 -5 0 0 0 S 0.0 0.0 7:26.83 kjournald
299 root 11 -5 0 0 0 S 0.0 0.0 0:00.00 kauditd
[root@centos-57_vm001 tmp]# tail -f /opt/couchbase/var/lib/couchbase/logs/errors.
errors.1 errors.2 errors.3 errors.idx errors.siz
[root@centos-57_vm001 tmp]# tail -f /opt/couchbase/var/lib/couchbase/logs/errors.
errors.1 errors.2 errors.3 errors.idx errors.siz
[root@centos-57_vm001 tmp]# tail -f /opt/couchbase/var/lib/couchbase/logs/errors.3
[{'stats_reader-default','ns_1@10.3.121.148'}

,

{latest,minute,1}]}}
Offender: [{pid,<0.3802.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]


[error_logger:error] [2012-04-26 13:50:07] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.3.121.148'},
{latest,minute,1}

]}}
Offender: [

{pid,<0.3820.0>}

,

{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]


[ns_server:error] [2012-04-26 13:50:10] [ns_1@10.3.121.148:<0.3667.0>:ns_janitor:do_cleanup:66] Bucket "default" not yet ready on ['ns_1@10.3.121.148']
[error_logger:error] [2012-04-26 13:50:10] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.3.121.148'},
{latest,minute,1}]}}
Offender: [{pid,<0.3839.0>},
{name,menelaus_web_alerts_srv}

,
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},

{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]


[error_logger:error] [2012-04-26 13:50:13] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.3.121.148'},
{latest,minute,1}]}}
Offender: [{pid,<0.3858.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent}

,

{shutdown,5000},
{child_type,worker}]


[error_logger:error] [2012-04-26 13:50:16] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.3.121.148'},
{latest,minute,1}]}}
Offender: [{pid,<0.3878.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000}

,

{child_type,worker}]


[error_logger:error] [2012-04-26 13:50:19] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.3.121.148'},
{latest,minute,1}]}}
Offender: [{pid,<0.3896.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}

]

[error_logger:error] [2012-04-26 13:50:22] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor:

{local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.3.121.148'},
{latest,minute,1}]}}
Offender: [{pid,<0.3924.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]


[error_logger:error] [2012-04-26 13:50:25] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}

Context: child_terminated
Reason: {noproc,
{gen_server,call,
[

{'stats_reader-default','ns_1@10.3.121.148'},
{latest,minute,1}]}}
Offender: [{pid,<0.3934.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]


[error_logger:error] [2012-04-26 13:50:25] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {timeout,{gen_server,call,
['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.3726.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]


[error_logger:error] [2012-04-26 13:50:28] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.3.121.148'}

,

{latest,minute,1}]}}
Offender: [{pid,<0.3964.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]


[error_logger:error] [2012-04-26 13:50:31] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.3.121.148'},
{latest,minute,1}

]}}
Offender: [

{pid,<0.3975.0>}

,

{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]


[error_logger:error] [2012-04-26 13:50:34] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.3.121.148'},
{latest,minute,1}]}}
Offender: [{pid,<0.4003.0>},
{name,menelaus_web_alerts_srv}

,
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},

{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]


[ns_server:error] [2012-04-26 13:50:35] [ns_1@10.3.121.148:<0.3867.0>:ns_janitor:do_cleanup:66] Bucket "default" not yet ready on ['ns_1@10.3.121.148']
[error_logger:error] [2012-04-26 13:50:35] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.3357.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['ns_memcached_sup-default','single_bucket_sup-default',
<0.3354.0>]
messages: [{'$gen_call',{<0.3368.0>,
{#Ref<0.0.9.57404>,'ns_1@10.3.121.148'}},
connected},
{'$gen_call',{<0.436.0>,#Ref<0.0.9.94429>},connected},
{'$gen_call',{<0.3449.0>,#Ref<0.0.9.109117>},topkeys},
{'$gen_call',{<0.3484.0>,
{#Ref<0.0.9.133059>,'ns_1@10.3.121.148'}},
connected},
{'$gen_call',{<0.436.0>,#Ref<0.0.9.162936>},connected},
{'$gen_call',{<0.3570.0>,
{#Ref<0.0.9.218693>,'ns_1@10.3.121.148'}},
connected},
{'$gen_call',{<0.436.0>,#Ref<0.0.9.231172>},connected},
{'$gen_call',{<0.436.0>,#Ref<0.0.9.237230>},connected},
{'$gen_call',{<0.3621.0>,
{#Ref<0.0.9.237251>,'ns_1@10.3.121.148'}},
connected},
{'$gen_call',{<0.436.0>,#Ref<0.0.9.237680>},connected},
{'$gen_call',{<0.3668.0>,
{#Ref<0.0.9.237823>,'ns_1@10.3.121.148'}},
connected},
{'$gen_call',{<0.436.0>,#Ref<0.0.9.238070>},connected},
{'$gen_call',{<0.3705.0>,
{#Ref<0.0.9.238269>,'ns_1@10.3.121.148'}},
connected},
{'$gen_call',{<0.436.0>,#Ref<0.0.9.238449>},connected},
{'$gen_call',{<0.3745.0>,
{#Ref<0.0.9.238776>,'ns_1@10.3.121.148'}},
connected},
{'$gen_call',{<0.436.0>,#Ref<0.0.9.238949>},connected},
{'$gen_call',{<0.3765.0>,#Ref<0.0.9.239062>},topkeys},
{'$gen_call',{<0.3784.0>,
{#Ref<0.0.9.239297>,'ns_1@10.3.121.148'}},
connected},
{'$gen_call',{<0.436.0>,#Ref<0.0.9.239331>},connected},
{'$gen_call',{<0.436.0>,#Ref<0.0.9.239720>},connected},
{'$gen_call',{<0.3829.0>,
{#Ref<0.0.9.239790>,'ns_1@10.3.121.148'}},
connected},
{'$gen_call',{<0.436.0>,#Ref<0.0.9.240174>},connected},
{'$gen_call',{<0.3868.0>,
{#Ref<0.0.9.240307>,'ns_1@10.3.121.148'}},
connected},
{'$gen_call',{<0.436.0>,#Ref<0.0.9.240548>},connected},
{'$gen_call',{<0.3905.0>,
{#Ref<0.0.9.240782>,'ns_1@10.3.121.148'}},
connected},
{'$gen_call',{<0.436.0>,#Ref<0.0.9.240961>},connected},
{'$gen_call',{<0.3943.0>,
{#Ref<0.0.9.241253>,'ns_1@10.3.121.148'}},
connected},
{'$gen_call',{<0.436.0>,#Ref<0.0.9.241440>},connected},
{'$gen_call',{<0.3984.0>,
{#Ref<0.0.9.241796>,'ns_1@10.3.121.148'}},
connected},
{'$gen_call',{<0.436.0>,#Ref<0.0.9.241834>},connected},
{'$gen_call',{<0.3993.0>,#Ref<0.0.9.241924>},topkeys},
{'$gen_call',{<0.436.0>,#Ref<0.0.9.242218>},connected},
{'$gen_call',{<0.4030.0>,
{#Ref<0.0.9.242287>,'ns_1@10.3.121.148'}},
connected}]
links: [#Port<0.8790>,<0.3356.0>,#Port<0.8794>,#Port<0.8783>,
#Port<0.8787>,#Port<0.8648>]
dictionary: []
trap_exit: true
status: running
heap_size: 196418
stack_size: 24
reductions: 18512
neighbours:

[error_logger:error] [2012-04-26 13:50:35] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'ns_memcached_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,stats,"default",0}},
{mfargs,{ns_memcached,start_link,[{"default",stats,0}]}},
{restart_type,permanent}

,

{shutdown,86400000}

,

{child_type,worker}]


[error_logger:error] [2012-04-26 13:50:35] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: shutdown
Offender: [{pid,undefined},
{name,{ns_memcached_sup,"default"}},
{mfargs,{ns_memcached_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]


[error_logger:error] [2012-04-26 13:50:35] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3354.0> on node 'ns_1@10.3.121.148' with exit value: badmatch,{error,shutdown,[{single_bucket_sup,'-start_link/1-fun-0-',2}]}


[error_logger:error] [2012-04-26 13:50:35] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.3965.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}

]

[error_logger:error] [2012-04-26 13:50:35] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor:

{local,ns_bucket_sup}

Context: child_terminated
Reason: badmatch,{error,shutdown,
[

{single_bucket_sup,'-start_link/1-fun-0-',2}

]}
Offender: [

{pid,<0.3354.0>}

,
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},

{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]


[error_logger:error] [2012-04-26 13:50:37] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.3.121.148'},
{latest,minute,1}]}}
Offender: [{pid,<0.4021.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent}

,

{shutdown,5000},
{child_type,worker}]


[stats:error] [2012-04-26 13:50:37] [ns_1@10.3.121.148:<0.458.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.3.121.148']
[stats:error] [2012-04-26 13:50:37] [ns_1@10.3.121.148:<0.446.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.3.121.148']
[error_logger:error] [2012-04-26 13:50:40] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.3.121.148'},
{latest,minute,1}]}}
Offender: [{pid,<0.4060.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000}

,

{child_type,worker}]


[error_logger:error] [2012-04-26 13:50:43] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.3.121.148'},
{latest,minute,1}]}}
Offender: [{pid,<0.4076.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}

]

[error_logger:error] [2012-04-26 13:50:46] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor:

{local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.3.121.148'},
{latest,minute,1}]}}
Offender: [{pid,<0.4097.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]


[error_logger:error] [2012-04-26 13:50:49] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}

Context: child_terminated
Reason: {noproc,
{gen_server,call,
[

{'stats_reader-default','ns_1@10.3.121.148'},
{latest,minute,1}]}}
Offender: [{pid,<0.4115.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]


[error_logger:error] [2012-04-26 13:50:52] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.3.121.148'}

,

{latest,minute,1}]}}
Offender: [{pid,<0.4142.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]


[error_logger:error] [2012-04-26 13:50:55] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.3.121.148'},
{latest,minute,1}

]}}
Offender: [

{pid,<0.4152.0>}

,

{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]


[stats:error] [2012-04-26 13:50:57] [ns_1@10.3.121.148:<0.458.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.3.121.148']
[error_logger:error] [2012-04-26 13:50:58] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.3.121.148'},
{latest,minute,1}]}}
Offender: [{pid,<0.4180.0>},
{name,menelaus_web_alerts_srv}

,
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},

{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]


[error_logger:error] [2012-04-26 13:51:01] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.3.121.148'},
{latest,minute,1}]}}
Offender: [{pid,<0.4192.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent}

,

{shutdown,5000},
{child_type,worker}]


[error_logger:error] [2012-04-26 13:51:04] [ns_1@10.3.121.148:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.3.121.148'},
{latest,minute,1}]}}
Offender: [{pid,<0.4219.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000}

,

{child_type,worker}

]

[root@centos-57_vm001 tmp]# top
[root@centos-57_vm001 tmp]# ps -aef | grep memcached
100 30158 30125 0 13:46 ? 00:00:00 /opt/couchbase/bin/memcached -X /opt/couchbase/lib/memcached/stdin_term_handler.so -l 0.0.0.0:11210,0.0.0.0:11209:1000 -E /opt/couchbase/lib/memcached/bucket_engine.so -B binary -r -c 10000 -e admin=_admin;default_bucket_name=default;auto_create=false
root 30184 29669 0 13:51 pts/0 00:00:00 grep memcached
[root@centos-57_vm001 tmp]# gdb -p
[root@centos-57_vm001 tmp]# top -p 30158

top - 13:51:32 up 7 days, 23:55, 1 user, load average: 0.22, 0.51, 0.26
Tasks: 1 total, 0 running, 1 sleeping, 0 stopped, 0 zombie
Cpu(s): 4.6%us, 1.7%sy, 0.0%ni, 92.9%id, 0.5%wa, 0.0%hi, 0.2%si, 0.2%st
Mem: 4194304k total, 1999608k used, 2194696k free, 496788k buffers
Swap: 0k total, 0k used, 0k free, 1144284k cached

PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
30158 couchbas 15 0 188m 59m 2432 S 0.0 1.4 0:00.13 memcached

[root@centos-57_vm001 tmp]# ps -aef | grep memcached
100 30158 30125 0 13:46 ? 00:00:00 /opt/couchbase/bin/memcached -X /opt/couchbase/lib/memcached/stdin_term_handler.so -l 0.0.0.0:11210,0.0.0.0:11209:1000 -E /opt/couchbase/lib/memcached/bucket_engine.so -B binary -r -c 10000 -e admin=_admin;default_bucket_name=default;auto_create=false
root 30187 29669 0 13:51 pts/0 00:00:00 grep memcached
[root@centos-57_vm001 tmp]#
[root@centos-57_vm001 tmp]# top -p 30125

top - 13:51:44 up 7 days, 23:55, 1 user, load average: 0.18, 0.49, 0.25
Tasks: 1 total, 0 running, 1 sleeping, 0 stopped, 0 zombie
Cpu(s): 0.0%us, 0.2%sy, 0.0%ni, 99.5%id, 0.0%wa, 0.0%hi, 0.0%si, 0.3%st
Mem: 4194304k total, 2006048k used, 2188256k free, 496788k buffers
Swap: 0k total, 0k used, 0k free, 1144692k cached

PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
30125 couchbas 25 0 251m 95m 6524 S 0.0 2.3 3:18.40 beam.smp

[root@centos-57_vm001 tmp]# ps -aef | grep memcached
100 30158 30125 0 13:46 ? 00:00:00 /opt/couchbase/bin/memcached -X /opt/couchbase/lib/memcached/stdin_term_handler.so -l 0.0.0.0:11210,0.0.0.0:11209:1000 -E /opt/couchbase/lib/memcached/bucket_engine.so -B binary -r -c 10000 -e admin=_admin;default_bucket_name=default;auto_create=false
root 30190 29669 0 13:51 pts/0 00:00:00 grep memcached
[root@centos-57_vm001 tmp]#
[root@centos-57_vm001 tmp]# top -p 30158

top - 13:52:02 up 7 days, 23:56, 1 user, load average: 0.13, 0.46, 0.25
Tasks: 1 total, 0 running, 1 sleeping, 0 stopped, 0 zombie
Cpu(s): 0.0%us, 0.0%sy, 0.0%ni, 99.8%id, 0.0%wa, 0.0%hi, 0.0%si, 0.2%st
Mem: 4194304k total, 2007064k used, 2187240k free, 496788k buffers
Swap: 0k total, 0k used, 0k free, 1145280k cached

PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
30158 couchbas 15 0 188m 59m 2432 S 0.0 1.4 0:00.13 memcached

[root@centos-57_vm001 tmp]# top
[root@centos-57_vm001 tmp]# gdb -p 30158
GNU gdb (GDB) CentOS (7.0.1-42.el5.centos)
Copyright (C) 2009 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>.
Attaching to process 30158
Reading symbols from /opt/couchbase/bin/memcached...done.
Reading symbols from /opt/couchbase/lib/memcached/libmemcached_utilities.so.0...done.
Loaded symbols for /opt/couchbase/lib/memcached/libmemcached_utilities.so.0
Reading symbols from /opt/couchbase/lib/libevent-2.0.so.5...done.
Loaded symbols for /opt/couchbase/lib/libevent-2.0.so.5
Reading symbols from /lib64/libdl.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libdl.so.2
Reading symbols from /lib64/libm.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib64/libm.so.6
Reading symbols from /lib64/librt.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/librt.so.1
Reading symbols from /opt/couchbase/lib/libtcmalloc_minimal.so.4...done.
Loaded symbols for /opt/couchbase/lib/libtcmalloc_minimal.so.4
Reading symbols from /lib64/libpthread.so.0...(no debugging symbols found)...done.
[Thread debugging using libthread_db enabled]
[New Thread 0x456a2940 (LWP 30174)]
[New Thread 0x44ca1940 (LWP 30173)]
[New Thread 0x442a0940 (LWP 30172)]
[New Thread 0x4389f940 (LWP 30171)]
[New Thread 0x42e9e940 (LWP 30170)]
[New Thread 0x41732940 (LWP 30169)]
[New Thread 0x40bc8940 (LWP 30168)]
[New Thread 0x4249d940 (LWP 30163)]
Loaded symbols for /lib64/libpthread.so.0
Reading symbols from /lib64/libc.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib64/libc.so.6
Reading symbols from /lib64/ld-linux-x86-64.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/ld-linux-x86-64.so.2
Reading symbols from /usr/lib64/libstdc++.so.6...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libstdc++.so.6
Reading symbols from /lib64/libgcc_s.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libgcc_s.so.1
Reading symbols from /opt/couchbase/lib/memcached/stdin_term_handler.so...done.
Loaded symbols for /opt/couchbase/lib/memcached/stdin_term_handler.so
Reading symbols from /opt/couchbase/lib/memcached/bucket_engine.so...done.
Loaded symbols for /opt/couchbase/lib/memcached/bucket_engine.so
Reading symbols from /opt/couchbase/lib/memcached/ep.so...done.
Loaded symbols for /opt/couchbase/lib/memcached/ep.so
Reading symbols from /opt/couchbase/lib/libcouchstore.so.1...done.
Loaded symbols for /opt/couchbase/lib/libcouchstore.so.1
Reading symbols from /opt/couchbase/lib/libsnappy.so.1...done.
Loaded symbols for /opt/couchbase/lib/libsnappy.so.1
Reading symbols from /lib64/libnss_files.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libnss_files.so.2

warning: no loadable sections found in added symbol-file system-supplied DSO at 0x7fffb6da9000
0x00002b609112a8a8 in epoll_wait () from /lib64/libc.so.6
(gdb) t a a bt

Thread 9 (Thread 0x4249d940 (LWP 30163)):
#0 0x00002b609111c78b in read () from /lib64/libc.so.6
#1 0x00002b60910c2d57 in _IO_new_file_underflow () from /lib64/libc.so.6
#2 0x00002b60910c371e in _IO_default_uflow_internal () from /lib64/libc.so.6
#3 0x00002b60910befdb in getc () from /lib64/libc.so.6
#4 0x00002b60918bf879 in check_stdin_thread (arg=0x4033f0) at extensions/daemon/stdin_check.c:19
#5 0x00002b6090e4173d in start_thread () from /lib64/libpthread.so.0
#6 0x00002b609112a4bd in clone () from /lib64/libc.so.6

Thread 8 (Thread 0x40bc8940 (LWP 30168)):
#0 0x00002b60910f0541 in nanosleep () from /lib64/libc.so.6
#1 0x00002b60910f0364 in sleep () from /lib64/libc.so.6
#2 0x0000000000415c58 in check_isasl_db_thread (arg=<value optimized out>) at daemon/isasl.c:233
#3 0x00002b6090e4173d in start_thread () from /lib64/libpthread.so.0
#4 0x00002b609112a4bd in clone () from /lib64/libc.so.6

Thread 7 (Thread 0x41732940 (LWP 30169)):
#0 0x00002b6090e48524 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00002b6090e43e1a in _L_lock_1034 () from /lib64/libpthread.so.0
#2 0x00002b6090e43cdc in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00002aaaaaaaf479 in must_lock (mutex=<value optimized out>) at bucket_engine.c:253
#4 0x00002aaaaaaaf794 in lock_engines (name=0x106c8068 "default") at bucket_engine.c:283
#5 find_bucket (name=0x106c8068 "default") at bucket_engine.c:569
#6 0x00002aaaaaab0ca4 in handle_connect (cookie=0x14ad4000, type=<value optimized out>, event_data=<value optimized out>, cb_data=0x2aaaaacb5480) at bucket_engine.c:1082
#7 0x0000000000403036 in perform_callbacks (type=ON_CONNECT, data=0x0, c=0x14ad4000) at daemon/memcached.c:198
#8 0x00000000004042cb in conn_new (sfd=51, parent_port=11209, init_state=<value optimized out>, event_flags=18, read_buffer_size=2048, transport=tcp_transport,
base=0x14b2e500, timeout=0x0) at daemon/memcached.c:890
#9 0x00000000004136d3 in thread_libevent_process (fd=<value optimized out>, which=<value optimized out>, arg=0x106ce500) at daemon/thread.c:342
#10 0x00002b6090326d0c in event_process_active_single_queue (base=0x14b2e500, flags=0) at event.c:1308
#11 event_process_active (base=0x14b2e500, flags=0) at event.c:1375
#12 event_base_loop (base=0x14b2e500, flags=0) at event.c:1572
#13 0x0000000000414074 in worker_libevent (arg=0x106ce500) at daemon/thread.c:305
#14 0x00002b6090e4173d in start_thread () from /lib64/libpthread.so.0
#15 0x00002b609112a4bd in clone () from /lib64/libc.so.6

Thread 6 (Thread 0x42e9e940 (LWP 30170)):
#0 0x00002b6090e48524 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00002b6090e43e1a in _L_lock_1034 () from /lib64/libpthread.so.0
#2 0x00002b6090e43cdc in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00002aaaaaaaf479 in must_lock (mutex=<value optimized out>) at bucket_engine.c:253
#4 0x00002aaaaaaaf794 in lock_engines (name=0x106c8068 "default") at bucket_engine.c:283
#5 find_bucket (name=0x106c8068 "default") at bucket_engine.c:569
--Type <return> to continue, or q <return> to quit--
#6 0x00002aaaaaab0ca4 in handle_connect (cookie=0x14ad42c0, type=<value optimized out>, event_data=<value optimized out>, cb_data=0x2aaaaacb5480) at bucket_engine.c:1082
#7 0x0000000000403036 in perform_callbacks (type=ON_CONNECT, data=0x0, c=0x14ad42c0) at daemon/memcached.c:198
#8 0x00000000004042cb in conn_new (sfd=52, parent_port=11209, init_state=<value optimized out>, event_flags=18, read_buffer_size=2048, transport=tcp_transport,
base=0x14b2e280, timeout=0x0) at daemon/memcached.c:890
#9 0x00000000004136d3 in thread_libevent_process (fd=<value optimized out>, which=<value optimized out>, arg=0x106ce600) at daemon/thread.c:342
#10 0x00002b6090326d0c in event_process_active_single_queue (base=0x14b2e280, flags=0) at event.c:1308
#11 event_process_active (base=0x14b2e280, flags=0) at event.c:1375
#12 event_base_loop (base=0x14b2e280, flags=0) at event.c:1572
#13 0x0000000000414074 in worker_libevent (arg=0x106ce600) at daemon/thread.c:305
#14 0x00002b6090e4173d in start_thread () from /lib64/libpthread.so.0
#15 0x00002b609112a4bd in clone () from /lib64/libc.so.6

Thread 5 (Thread 0x4389f940 (LWP 30171)):

#0 0x00002b60910f0541 in nanosleep () from /lib64/libc.so.6
#1 0x00002b6091123ed4 in usleep () from /lib64/libc.so.6
#2 0x00002aaaaadb7c4b in MemcachedEngine::ensureConnection (this=0x14bbe000) at mc-kvstore/mc-engine.cc:776
#3 0x00002aaaaadb82e3 in MemcachedEngine::sendCommand (this=0x14bbe000, rh=0x14bbadc0) at mc-kvstore/mc-engine.cc:873
#4 0x00002aaaaadb86de in MemcachedEngine::selectBucket (this=0x14bbe000) at mc-kvstore/mc-engine.cc:1325
#5 0x00002aaaaadb98cf in MemcachedEngine::MemcachedEngine (this=0x14bbe000, e=<value optimized out>, config=<value optimized out>) at mc-kvstore/mc-engine.cc:566
#6 0x00002aaaaadc299a in CouchKVStore::open (this=0x14bb8000) at couch-kvstore/couch-kvstore.cc:730
#7 0x00002aaaaadc9a19 in CouchKVStore::CouchKVStore (this=0x14bb8000, theEngine=<value optimized out>) at couch-kvstore/couch-kvstore.cc:187
#8 0x00002aaaaad9c97b in KVStoreFactory::create (theEngine=...) at kvstore.cc:29
#9 0x00002aaaaad4eba5 in EventuallyPersistentEngine::initialize (this=0x14ba0000, config=<value optimized out>) at ep_engine.cc:1271
#10 0x00002aaaaad4f176 in EvpInitialize (handle=0x14ba0000,
config_str=0x4389e4c3 "ht_size=3079;ht_locks=5;tap_noop_interval=20;max_txn_size=10000;max_size=2290089984;tap_keepalive=300;dbname=/opt/couchbase/var/lib/couchdb/default;allow_data_loss_during_shutdown=true;backend=couchdb"...) at ep_engine.cc:116
#11 0x00002aaaaaaafcf0 in create_bucket_UNLOCKED (e=0x2aaaaacb5480, bucket_name=0x4389e650 "default", path=0x4389e4a0 "/opt/couchbase/lib/memcached/ep.so",
config=0x4389e4c3 "ht_size=3079;ht_locks=5;tap_noop_interval=20;max_txn_size=10000;max_size=2290089984;tap_keepalive=300;dbname=/opt/couchbase/var/lib/couchdb/default;allow_data_loss_during_shutdown=true;backend=couchdb"..., e_out=0x0, msg=0x4389e090 "", msglen=1024) at bucket_engine.c:705
#12 0x00002aaaaaab0814 in handle_create_bucket (handle=0x2aaaaacb5480, cookie=0x14afd8c0, request=<value optimized out>, response=0x406110 <binary_response_handler>)
at bucket_engine.c:2113
#13 0x00002aaaaaab0a25 in bucket_unknown_command (handle=0x2aaaaacb5480, cookie=0x14afd8c0, request=0x14aee800, response=0x406110 <binary_response_handler>)
at bucket_engine.c:2379
#14 0x000000000041090b in process_bin_unknown_packet (c=0x14afd8c0) at daemon/memcached.c:2875
#15 process_bin_packet (c=0x14afd8c0) at daemon/memcached.c:3130
#16 complete_nread_binary (c=0x14afd8c0) at daemon/memcached.c:3691
#17 complete_nread (c=0x14afd8c0) at daemon/memcached.c:3773
#18 conn_nread (c=0x14afd8c0) at daemon/memcached.c:5613
#19 0x00000000004070fc in event_handler (fd=<value optimized out>, which=<value optimized out>, arg=0x14afd8c0) at daemon/memcached.c:5922
#20 0x00002b6090326d0c in event_process_active_single_queue (base=0x14b2ec80, flags=0) at event.c:1308
#21 event_process_active (base=0x14b2ec80, flags=0) at event.c:1375
#22 event_base_loop (base=0x14b2ec80, flags=0) at event.c:1572
#23 0x0000000000414074 in worker_libevent (arg=0x106ce700) at daemon/thread.c:305
--Type <return> to continue, or q <return> to quit--
#24 0x00002b6090e4173d in start_thread () from /lib64/libpthread.so.0
#25 0x00002b609112a4bd in clone () from /lib64/libc.so.6

Thread 4 (Thread 0x442a0940 (LWP 30172)):
#0 0x00002b6090e48524 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00002b6090e43e1a in _L_lock_1034 () from /lib64/libpthread.so.0
#2 0x00002b6090e43cdc in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00002aaaaaaaf479 in must_lock (mutex=<value optimized out>) at bucket_engine.c:253
#4 0x00002aaaaaaaf794 in lock_engines (name=0x106c8068 "default") at bucket_engine.c:283
#5 find_bucket (name=0x106c8068 "default") at bucket_engine.c:569
#6 0x00002aaaaaab0ca4 in handle_connect (cookie=0x14afdb80, type=<value optimized out>, event_data=<value optimized out>, cb_data=0x2aaaaacb5480) at bucket_engine.c:1082
#7 0x0000000000403036 in perform_callbacks (type=ON_CONNECT, data=0x0, c=0x14afdb80) at daemon/memcached.c:198
#8 0x00000000004042cb in conn_new (sfd=50, parent_port=11209, init_state=<value optimized out>, event_flags=18, read_buffer_size=2048, transport=tcp_transport,
base=0x14b2ea00, timeout=0x0) at daemon/memcached.c:890
#9 0x00000000004136d3 in thread_libevent_process (fd=<value optimized out>, which=<value optimized out>, arg=0x106ce800) at daemon/thread.c:342
#10 0x00002b6090326d0c in event_process_active_single_queue (base=0x14b2ea00, flags=0) at event.c:1308
#11 event_process_active (base=0x14b2ea00, flags=0) at event.c:1375
#12 event_base_loop (base=0x14b2ea00, flags=0) at event.c:1572
#13 0x0000000000414074 in worker_libevent (arg=0x106ce800) at daemon/thread.c:305
#14 0x00002b6090e4173d in start_thread () from /lib64/libpthread.so.0
#15 0x00002b609112a4bd in clone () from /lib64/libc.so.6

Thread 3 (Thread 0x44ca1940 (LWP 30173)):
#0 0x00002b609112a8a8 in epoll_wait () from /lib64/libc.so.6
#1 0x00002b6090337ba6 in epoll_dispatch (base=0x14b2e780, tv=<value optimized out>) at epoll.c:404
#2 0x00002b6090326c11 in event_base_loop (base=0x14b2e780, flags=0) at event.c:1558
#3 0x0000000000414074 in worker_libevent (arg=0x106ce900) at daemon/thread.c:305
#4 0x00002b6090e4173d in start_thread () from /lib64/libpthread.so.0
#5 0x00002b609112a4bd in clone () from /lib64/libc.so.6

Thread 2 (Thread 0x456a2940 (LWP 30174)):
#0 0x00002b60910f0541 in nanosleep () from /lib64/libc.so.6
#1 0x00002b6091123ed4 in usleep () from /lib64/libc.so.6
#2 0x00002aaaaad6e312 in updateStatsThread (arg=0x106cc7c0) at memory_tracker.cc:31
#3 0x00002b6090e4173d in start_thread () from /lib64/libpthread.so.0
#4 0x00002b609112a4bd in clone () from /lib64/libc.so.6

Thread 1 (Thread 0x2b60918be240 (LWP 30158)):
#0 0x00002b609112a8a8 in epoll_wait () from /lib64/libc.so.6
#1 0x00002b6090337ba6 in epoll_dispatch (base=0x14b2e000, tv=<value optimized out>) at epoll.c:404
#2 0x00002b6090326c11 in event_base_loop (base=0x14b2e000, flags=0) at event.c:1558
#3 0x00000000004095b2 in main (argc=<value optimized out>, argv=<value optimized out>) at daemon/memcached.c:7900
(gdb)
(gdb)
(gdb)