command 'couchbase-server restart' hangs when node is out of disk space

got it when verified MB-7602

[root@localhost tmp]# dd if=/dev/zero of=fake1trt5dyrfgfhf bs=3G count=1 &
[2] 15178
[root@localhost tmp]# df
Filesystem 1K-blocks Used Available Use% Mounted on
/dev/mapper/VolGroup00-LogVol00
55007284 52154868 13072 100% /
/dev/sda1 101086 19983 75884 21% /boot
tmpfs 1972692 0 1972692 0% /dev/shm
[root@localhost tmp]# 0+1 records in
0+1 records out
2147479552 bytes (2.1 GB) copied, 6.65531 seconds, 323 MB/s

[2]- Done dd if=/dev/zero of=fake1trt5dyrfgfhf bs=3G count=1
[root@localhost tmp]# ps -ef | grep couchbase
500 14528 1 0 Apr25 ? 00:00:00 /opt/couchbase/lib/erlang/erts-5.8.5/bin/epmd -daemon
500 14544 1 0 Apr25 ? 00:00:50 /opt/couchbase/lib/erlang/erts-5.8.5/bin/beam.smp -A 16 – -root /opt/couchbase/lib/erlang -progname erl – -home /home/couchbase – -smp enable -kernel inet_dist_listen_min 21100 inet_dist_listen_max 21299 error_logger false -sasl sasl_error_logger false -name babysitter_of_ns_1@127.0.0.1 -setcookie nocookie -noshell -noinput -noshell -noinput -eval os:putenv("ERL_MAX_PORTS",""). -run ns_babysitter_bootstrap – -couch_ini /opt/couchbase/etc/couchdb/default.ini /opt/couchbase/etc/couchdb/default.d/capi.ini /opt/couchbase/etc/couchdb/default.d/geocouch.ini /opt/couchbase/etc/couchdb/local.ini -ns_babysitter cookiefile "/opt/couchbase/var/lib/couchbase/couchbase-server.cookie" -ns_server config_path "/opt/couchbase/etc/couchbase/static_config" -ns_server pidfile "/opt/couchbase/var/lib/couchbase/couchbase-server.pid" -ns_server cookiefile "/opt/couchbase/var/lib/couchbase/couchbase-server.cookie-ns-server" -ns_server enable_mlockall true
500 14569 14544 2 Apr25 ? 02:15:19 /opt/couchbase/lib/erlang/erts-5.8.5/bin/beam.smp -A 16 -sbt u -P 327680 -K true -MMmcs 30 – -root /opt/couchbase/lib/erlang -progname erl – -home /home/couchbase – -smp enable -setcookie nocookie -kernel inet_dist_listen_min 21100 inet_dist_listen_max 21299 error_logger false -sasl sasl_error_logger false -nouser -run child_erlang child_start ns_bootstrap – -smp enable -kernel inet_dist_listen_min 21100 inet_dist_listen_max 21299 error_logger false -sasl sasl_error_logger false -couch_ini /opt/couchbase/etc/couchdb/default.ini /opt/couchbase/etc/couchdb/default.d/capi.ini /opt/couchbase/etc/couchdb/default.d/geocouch.ini /opt/couchbase/etc/couchdb/local.ini -ns_babysitter cookiefile "/opt/couchbase/var/lib/couchbase/couchbase-server.cookie" -ns_server config_path "/opt/couchbase/etc/couchbase/static_config" -ns_server pidfile "/opt/couchbase/var/lib/couchbase/couchbase-server.pid" -ns_server cookiefile "/opt/couchbase/var/lib/couchbase/couchbase-server.cookie-ns-server" -ns_server enable_mlockall true
500 14599 14569 0 Apr25 ? 00:00:08 /opt/couchbase/lib/erlang/lib/os_mon-2.2.7/priv/bin/memsup
500 14600 14569 0 Apr25 ? 00:00:00 /opt/couchbase/lib/erlang/lib/os_mon-2.2.7/priv/bin/cpu_sup
500 14601 14569 0 Apr25 ? 00:00:00 /opt/couchbase/lib/erlang/lib/ssl-4.1.6/priv/bin/ssl_esock
500 14605 14569 0 Apr25 ? 00:00:59 /opt/couchbase/lib/ns_server/erlang/lib/ns_server-2.0.1_linux_179_g3d51cec/priv/i386-linux-godu
500 14608 14544 0 Apr25 ? 00:12:05 /opt/couchbase/bin/memcached -X /opt/couchbase/lib/memcached/stdin_term_handler.so -X /opt/couchbase/lib/memcached/file_logger.so,cyclesize=104857600;sleeptime=19;filename=/opt/couchbase/var/lib/couchbase/logs/memcached.log -l 0.0.0.0:11210,0.0.0.0:11209:1000 -p 11210 -E /opt/couchbase/lib/memcached/bucket_engine.so -B binary -r -c 10000 -e admin=_admin;default_bucket_name=default;auto_create=false
500 14684 14544 0 05:46 ? 00:00:00 /opt/couchbase/bin/moxi -Z port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200 -z url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming -p 0 -Y y -O stderr
root 15183 14848 0 06:44 pts/1 00:00:00 grep couchbase
[root@localhost tmp]# sudo /etc/init.d/couchbase-server restart
Stopping couchbase-server

(HANGS HERE)

after about 20 minutes from another terminal I see that 'couchbase-server restart' hanged

[root@localhost ~]# ps -ef| grep couchbase
500 14528 1 0 Apr25 ? 00:00:00 /opt/couchbase/lib/erlang/erts-5.8.5/bin/epmd -daemon
500 14544 1 0 Apr25 ? 00:00:50 /opt/couchbase/lib/erlang/erts-5.8.5/bin/beam.smp -A 16 – -root /opt/couchbase/lib/erlang -progname erl – -home /home/couchbase – -smp enable -kernel inet_dist_listen_min 21100 inet_dist_listen_max 21299 error_logger false -sasl sasl_error_logger false -name babysitter_of_ns_1@127.0.0.1 -setcookie nocookie -noshell -noinput -noshell -noinput -eval os:putenv("ERL_MAX_PORTS",""). -run ns_babysitter_bootstrap – -couch_ini /opt/couchbase/etc/couchdb/default.ini /opt/couchbase/etc/couchdb/default.d/capi.ini /opt/couchbase/etc/couchdb/default.d/geocouch.ini /opt/couchbase/etc/couchdb/local.ini -ns_babysitter cookiefile "/opt/couchbase/var/lib/couchbase/couchbase-server.cookie" -ns_server config_path "/opt/couchbase/etc/couchbase/static_config" -ns_server pidfile "/opt/couchbase/var/lib/couchbase/couchbase-server.pid" -ns_server cookiefile "/opt/couchbase/var/lib/couchbase/couchbase-server.cookie-ns-server" -ns_server enable_mlockall true
500 14569 14544 2 Apr25 ? 02:16:03 /opt/couchbase/lib/erlang/erts-5.8.5/bin/beam.smp -A 16 -sbt u -P 327680 -K true -MMmcs 30 – -root /opt/couchbase/lib/erlang -progname erl – -home /home/couchbase – -smp enable -setcookie nocookie -kernel inet_dist_listen_min 21100 inet_dist_listen_max 21299 error_logger false -sasl sasl_error_logger false -nouser -run child_erlang child_start ns_bootstrap – -smp enable -kernel inet_dist_listen_min 21100 inet_dist_listen_max 21299 error_logger false -sasl sasl_error_logger false -couch_ini /opt/couchbase/etc/couchdb/default.ini /opt/couchbase/etc/couchdb/default.d/capi.ini /opt/couchbase/etc/couchdb/default.d/geocouch.ini /opt/couchbase/etc/couchdb/local.ini -ns_babysitter cookiefile "/opt/couchbase/var/lib/couchbase/couchbase-server.cookie" -ns_server config_path "/opt/couchbase/etc/couchbase/static_config" -ns_server pidfile "/opt/couchbase/var/lib/couchbase/couchbase-server.pid" -ns_server cookiefile "/opt/couchbase/var/lib/couchbase/couchbase-server.cookie-ns-server" -ns_server enable_mlockall true
500 14599 14569 0 Apr25 ? 00:00:08 /opt/couchbase/lib/erlang/lib/os_mon-2.2.7/priv/bin/memsup
500 14600 14569 0 Apr25 ? 00:00:00 /opt/couchbase/lib/erlang/lib/os_mon-2.2.7/priv/bin/cpu_sup
500 14601 14569 0 Apr25 ? 00:00:00 /opt/couchbase/lib/erlang/lib/ssl-4.1.6/priv/bin/ssl_esock
500 14605 14569 0 Apr25 ? 00:00:59 /opt/couchbase/lib/ns_server/erlang/lib/ns_server-2.0.1_linux_179_g3d51cec/priv/i386-linux-godu
500 14608 14544 0 Apr25 ? 00:28:06 /opt/couchbase/bin/memcached -X /opt/couchbase/lib/memcached/stdin_term_handler.so -X /opt/couchbase/lib/memcached/file_logger.so,cyclesize=104857600;sleeptime=19;filename=/opt/couchbase/var/lib/couchbase/logs/memcached.log -l 0.0.0.0:11210,0.0.0.0:11209:1000 -p 11210 -E /opt/couchbase/lib/memcached/bucket_engine.so -B binary -r -c 10000 -e admin=_admin;default_bucket_name=default;auto_create=false
root 15184 14848 0 06:44 pts/1 00:00:00 /bin/sh /etc/init.d/couchbase-server restart
root 15188 15184 0 06:44 pts/1 00:00:00 /bin/sh /opt/couchbase/bin/couchbase-server -k
root 15192 15188 0 06:44 pts/1 00:00:00 /opt/couchbase/lib/erlang/erts-5.8.5/bin/beam.smp – -root /opt/couchbase/lib/erlang -progname erl – -home /root – -name executioner@executioner -noshell -hidden -setcookie naennqlykmogvirw -eval ns_babysitter_bootstrap:remote_stop('babysitter_of_ns_1@127.0.0.1')
root 15305 15217 0 07:00 pts/2 00:00:00 grep couchbase

trying to stop restart:

(Ctrl+c gives)
BREAK: (a)bort (c)ontinue (p)roc info (i)nfo (l)oaded
(v)ersion (k)ill (D)b-tables (d)istribution
(again Ctrl+c)
Starting couchbase-server [ OK ]

but couchbase was not restarted ( the same process ids)