Details
-
Bug
-
Resolution: Fixed
-
Critical
-
6.5.0
-
Linux couch01 4.19.0-6-amd64 #1 SMP Debian 4.19.67-2+deb10u1 (2019-09-20) x86_64 GNU/Linux
-
Untriaged
-
Unknown
Description
Since last Sat/Sun Mad-Hatter source builds no longer responds correctly to "couchbas-server -k" as in this does not shut down the server.
This used to shut down the instance completely (about 8-10 lines to the console)
~/couch/install/bin/couchbase-server -k
|
But the above no longer does so in the last several days we get a 100+lines (shown in the details) and the couchbase instance remains running.
As such I have had to resort to the following e.g. killing processes (a very brutal methodology)
kill `ps -elf | grep erl | egrep ' erl_child'|'epmd -d' | awk -e '\{print $4}' | paste -d " " - - - - -`
|
Details
linuxbrew@couch01:~/scale_it$ ~/couch/install/bin/couchbase-server -k
|
2019-11-21 19:11:22 cb_dist: terminating with reason: {noproc,
|
{gen_server,call,
|
[kernel_safe_sup,
|
{start_child,
|
{timer_server,
|
{timer,start_link,[]},
|
permanent,1000,worker,
|
[timer]}},
|
infinity]}}
|
2019-11-21 19:11:22 ** Generic server ~tp terminating
|
** Last message in was ~tp~n** When Server state == ~tp~n** Reason for termination == ~n** ~tp~n** Client ~tp stacktrace~n** ~tp~n
|
cb_dist
|
{accept,<0.57.0>}
|
{s,[],[],1,undefined,[{external_listeners,[inet_tcp_dist,inet6_tcp_dist]},{preferred_external_proto,inet_tcp_dist},{preferred_local_proto,inet_tcp_dist}],executioner,undefined,[]}
|
{{noproc,{gen_server,call,[kernel_safe_sup,{start_child,{timer_server,{timer,start_link,[]},permanent,1000,worker,[timer]}},infinity]}},[{gen_server,call,3,[{file,"gen_server.erl"},{line,214}]},{timer,ensure_started,0,[{file,"timer.erl"},{line,265}]},{timer,req,2,[{file,"timer.erl"},{line,274}]},{cb_dist,start_ensure_config_timer,1,[{file,"src/cb_dist.erl"},{line,512}]},{lists,foldl,3,[{file,"lists.erl"},{line,1263}]},{cb_dist,handle_call,3,[{file,"src/cb_dist.erl"},{line,294}]},{gen_server,try_handle_call,4,[{file,"gen_server.erl"},{line,636}]},{gen_server,handle_msg,6,[{file,"gen_server.erl"},{line,665}]}]}
|
net_kernel
|
[{gen,do_call,4,[{file,"gen.erl"},{line,169}]},{gen_server,call,3,[{file,"gen_server.erl"},{line,210}]},{net_kernel,start_protos,5,[{file,"net_kernel.erl"},{line,1404}]},{net_kernel,start_protos,4,[{file,"net_kernel.erl"},{line,1388}]},{net_kernel,init_node,3,[{file,"net_kernel.erl"},{line,1246}]},{net_kernel,init,1,[{file,"net_kernel.erl"},{line,374}]},{gen_server,init_it,2,[{file,"gen_server.erl"},{line,365}]},{gen_server,init_it,6,[{file,"gen_server.erl"},{line,333}]}]
|
2019-11-21 19:11:22 crash_report
|
initial_call: {cb_dist,init,['Argument__1']}
|
pid: <0.54.0>
|
registered_name: cb_dist
|
error_info: {exit,{noproc,{gen_server,call,[kernel_safe_sup,{start_child,{timer_server,{timer,start_link,[]},permanent,1000,worker,[timer]}},infinity]}},[{gen_server,call,3,[{file,"gen_server.erl"},{line,214}]},{timer,ensure_started,0,[{file,"timer.erl"},{line,265}]},{timer,req,2,[{file,"timer.erl"},{line,274}]},{cb_dist,start_ensure_config_timer,1,[{file,"src/cb_dist.erl"},{line,512}]},{lists,foldl,3,[{file,"lists.erl"},{line,1263}]},{cb_dist,handle_call,3,[{file,"src/cb_dist.erl"},{line,294}]},{gen_server,try_handle_call,4,[{file,"gen_server.erl"},{line,636}]},{gen_server,handle_msg,6,[{file,"gen_server.erl"},{line,665}]}]}
|
ancestors: [net_sup,kernel_sup,<0.36.0>]
|
message_queue_len: 0
|
messages: []
|
links: [<0.45.0>]
|
dictionary: [{rand_seed,{#{bits=>58,jump=>#Fun<rand.8.15449617>,next=>#Fun<rand.5.15449617>,type=>exrop,uniform=>#Fun<rand.6.15449617>,uniform_n=>#Fun<rand.7.15449617>,weak_low_bits=>1},[247711435958569175|13558007168323293]}}]
|
trap_exit: true
|
status: running
|
heap_size: 1598
|
stack_size: 27
|
reductions: 4488
|
2019-11-21 19:11:22 crash_report
|
initial_call: {net_kernel,init,['Argument__1']}
|
pid: <0.57.0>
|
registered_name: []
|
error_info: {exit,{{noproc,{gen_server,call,[kernel_safe_sup,{start_child,{timer_server,{timer,start_link,[]},permanent,1000,worker,[timer]}},infinity]}},{gen_server,call,[cb_dist,{accept,<0.57.0>},infinity]}},[{gen_server,call,3,[{file,"gen_server.erl"},{line,214}]},{net_kernel,start_protos,5,[{file,"net_kernel.erl"},{line,1404}]},{net_kernel,start_protos,4,[{file,"net_kernel.erl"},{line,1388}]},{net_kernel,init_node,3,[{file,"net_kernel.erl"},{line,1246}]},{net_kernel,init,1,[{file,"net_kernel.erl"},{line,374}]},{gen_server,init_it,2,[{file,"gen_server.erl"},{line,365}]},{gen_server,init_it,6,[{file,"gen_server.erl"},{line,333}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,247}]}]}
|
ancestors: [net_sup,kernel_sup,<0.36.0>]
|
message_queue_len: 0
|
messages: []
|
links: [<0.45.0>]
|
dictionary: [{longnames,true}]
|
trap_exit: true
|
status: running
|
heap_size: 987
|
stack_size: 27
|
reductions: 401
|
2019-11-21 19:11:22 supervisor_report
|
supervisor: {local,net_sup}
|
errorContext: start_error
|
reason: {'EXIT',nodistribution}
|
offender: [{pid,undefined},{id,net_kernel},{mfargs,{net_kernel,start_link,[['executioner@cb.local',longnames],true]}},{restart_type,permanent},{shutdown,2000},{child_type,worker}]
|
2019-11-21 19:11:22 supervisor_report
|
supervisor: {local,net_sup}
|
errorContext: shutdown_error
|
reason: {noproc,{gen_server,call,[kernel_safe_sup,{start_child,{timer_server,{timer,start_link,[]},permanent,1000,worker,[timer]}},infinity]}}
|
offender: [{pid,<0.54.0>},{id,cb_dist},{mfargs,{cb_dist,start_link,[]}},{restart_type,permanent},{shutdown,infinity},{child_type,worker}]
|
2019-11-21 19:11:22 supervisor_report
|
supervisor: {local,kernel_sup}
|
errorContext: start_error
|
reason: {shutdown,{failed_to_start_child,net_kernel,{'EXIT',nodistribution}}}
|
offender: [{pid,undefined},{id,net_sup},{mfargs,{erl_distribution,start_link,[]}},{restart_type,permanent},{shutdown,infinity},{child_type,supervisor}]
|
2019-11-21 19:11:22 crash_report
|
initial_call: {application_master,init,['Argument__1','Argument__2','Argument__3','Argument__4']}
|
pid: <0.35.0>
|
registered_name: []
|
error_info: {exit,{{shutdown,{failed_to_start_child,net_sup,{shutdown,{failed_to_start_child,net_kernel,{'EXIT',nodistribution}}}}},{kernel,start,[normal,[]]}},[{application_master,init,4,[{file,"application_master.erl"},{line,134}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,247}]}]}
|
ancestors: [<0.34.0>]
|
message_queue_len: 1
|
messages: [{'EXIT',<0.36.0>,normal}]
|
links: [<0.34.0>,<0.33.0>]
|
dictionary: []
|
trap_exit: true
|
status: running
|
heap_size: 376
|
stack_size: 27
|
reductions: 214
|
2019-11-21 19:11:22 std_info
|
application: kernel
|
exited: {{shutdown,{failed_to_start_child,net_sup,{shutdown,{failed_to_start_child,net_kernel,{'EXIT',nodistribution}}}}},{kernel,start,[normal,[]]}}
|
type: permanent
|
{"Kernel pid terminated",application_controller,"{application_start_failure,kernel,{{shutdown,{failed_to_start_child,net_sup,{shutdown,{failed_to_start_child,net_kernel,{'EXIT',nodistribution}}}}},{kernel,start,[normal,[]]}}}"}
|
Kernel pid terminated (application_controller) ({application_start_failure,kernel,{{shutdown,{failed_to_start_child,net_sup,{shutdown,{failed_to_start_child,net_kernel,{'EXIT',nodistribution}}}}},{ker
|
|
|
Crash dump is being written to: erl_crash.dump.1574392281.17162.babysitter...done
|
linuxbrew@couch01:~/scale_it$ sleep 10
|
linuxbrew@couch01:~/scale_it$ top -b -n 1 -u linuxbrew |wc -l
|
46
|
linuxbrew@couch01:~/scale_it$ kill `ps -elf | grep erl | egrep ' erl_child'\|'epmd -d' | awk -e '{print $4}' | paste -d " " - - - - -`
|
-bash: kill: (17260) - No such process
|
linuxbrew@couch01:~/scale_it$
|
linuxbrew@couch01:~/scale_it$
|
linuxbrew@couch01:~/scale_it$ top -b -n 1 -u linuxbrew |wc -l
|
10
|
linuxbrew@couch01:~/scale_it$ top -b -n 1 -u linuxbrew |wc -l
|
10
|
linuxbrew@couch01:~/scale_it$ top -b -n 1 -u linuxbrew
|
top - 19:12:33 up 1 day, 4:57, 1 user, load average: 0.60, 0.52, 0.63
|
Tasks: 238 total, 1 running, 237 sleeping, 0 stopped, 0 zombie
|
%Cpu(s): 0.2 us, 0.2 sy, 0.0 ni, 99.5 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st
|
MiB Mem : 64319.3 total, 61382.1 free, 386.4 used, 2550.7 buff/cache
|
MiB Swap: 65433.0 total, 65433.0 free, 0.0 used. 62106.6 avail Mem
|
|
|
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
|
17272 linuxbr+ 20 0 11092 3580 3020 R 12.5 0.0 0:00.02 top
|
17075 linuxbr+ 20 0 8044 4908 3312 S 0.0 0.0 0:00.11 bash
|
linuxbrew@couch01:~/scale_it$
|