{noformat} root@node3-cb700-beta-ubuntu20:/opt/couchbase/var/lib/couchbase/logs# tail -f babysitter.log ns_server<0.185.0>: [{file,"gen_statem.erl"},{line,1120}]}, ns_server<0.185.0>: {proc_lib,init_p_do_apply,3, ns_server<0.185.0>: [{file,"proc_lib.erl"},{line,249}]}]}}}}}}}}}, ns_server<0.185.0>: {ns_server,start,[normal,[]]}}}} ns_server<0.185.0>: [{ns_bootstrap,start,0,[{file,"src/ns_bootstrap.erl"},{line,31}]}, ns_server<0.185.0>: {child_erlang,do_child_start,1,[{file,"src/child_erlang.erl"},{line,105}]}, ns_server<0.185.0>: {child_erlang,child_start,1,[{file,"src/child_erlang.erl"},{line,83}]}, ns_server<0.185.0>: {init,start_em,1,[]}, ns_server<0.185.0>: {init,do_boot,3,[]}] [error_logger:error,2021-06-03T09:43:22.305Z,babysitter_of_ns_1@cb.local:<0.185.0>:ale_error_logger_handler:do_log:101] =========================ERROR REPORT========================= ** Generic server <0.185.0> terminating ** Last message in was {#Port<0.32>,{exit_status,3}} ** When Server state == {state,#Port<0.32>, {ns_server,"/opt/couchbase/lib/erlang/bin/erl", ["+A","16","-smp","enable","+sbt","u","+P", "327680","+K","true","+swt","low","+sbwt", "none","+MMmcs","30","+e102400","-setcookie", "nocookie","-kernel","logger", "[{handler, default, undefined}]","-user", "user_io","-run","child_erlang", "child_start","ns_bootstrap","--","-smp", "enable","-kernel","logger", "[{handler, default, undefined}]","inetrc", "\"/opt/couchbase/etc/couchbase/hosts.cfg\"", "dist_config_file", "\"/opt/couchbase/var/lib/couchbase/config/dist_cfg\"", "-proto_dist","cb","-epmd_module","cb_epmd", "-ssl_dist_optfile", "/opt/couchbase/etc/couchbase/ssl_dist_opts", "-couch_ini", "/opt/couchbase/etc/couchdb/default.ini", "/opt/couchbase/etc/couchdb/default.d/capi.ini", "/opt/couchbase/etc/couchdb/default.d/geocouch.ini", "/opt/couchbase/etc/couchdb/local.ini"], [{env, [{"NS_SERVER_BABYSITTER_PID","4690"}, {"CHILD_ERLANG_ENV_ARGS", "[{ns_babysitter,\n [{cookiefile,\n \"/opt/couchbase/var/lib/couchbase/couchbase-server.babysitter.cookie\"},\n {pidfile,\"/opt/couchbase/var/lib/couchbase/couchbase-server.pid\"},\n {nodefile,\n \"/opt/couchbase/var/lib/couchbase/couchbase-server.babysitter.node\"}]},\n {ns_server,\n [{loglevel_ns_doctor,debug},\n {config_path,\"/opt/couchbase/etc/couchbase/static_config\"},\n {max_r,20},\n {loglevel_xdcr,debug},\n {error_logger_mf_dir,\"/opt/couchbase/var/lib/couchbase/logs\"},\n {loglevel_menelaus,debug},\n {nodefile,\"/opt/couchbase/var/lib/couchbase/couchbase-server.node\"},\n {loglevel_ns_server,debug},\n {disk_sink_opts,\n [{rotation,\n [{compress,true},\n {size,41943040},\n {num_files,10},\n {buffer_size_max,52428800}]}]},\n {loglevel_cluster,debug},\n {max_t,10},\n {loglevel_rebalance,debug},\n {path_config_libdir,\"/opt/couchbase/lib\"},\n {net_kernel_verbosity,10},\n {loglevel_mapreduce_errors,debug},\n {loglevel_views,debug},\n {loglevel_default,debug},\n {path_config_bindir,\"/opt/couchbase/bin\"},\n {path_config_tmpdir,\"/opt/couchbase/var/lib/couchbase/tmp\"},\n {loglevel_stats,debug},\n {loglevel_user,debug},\n {disk_sink_opts_json_rpc,\n [{rotation,\n [{compress,true},\n {size,41943040},\n {num_files,2},\n {buffer_size_max,52428800}]}]},\n {loglevel_couchdb,info},\n {path_config_secdir,\"/opt/couchbase/etc/security\"},\n {loglevel_access,info},\n {path_config_etcdir,\"/opt/couchbase/etc/couchbase\"},\n {path_config_datadir,\"/opt/couchbase/var/lib/couchbase\"},\n {loglevel_cbas,debug},\n {loglevel_error_logger,debug}]}]"}, {"ERL_CRASH_DUMP", "erl_crash.dump.1622713300.4690.ns_server"}]}, exit_status,use_stdio,stream,eof]}, {ringbuffer,1077,1024, {[{<<" {init,do_boot,3,[]}]">>,21}, {<<" {init,start_em,1,[]},">>,22}, {<<" {child_erlang,child_start,1,[{file,\"src/child_erlang.erl\"},{line,83}]},">>, 72}, {<<" {child_erlang,do_child_start,1,[{file,\"src/child_erlang.erl\"},{line,105}]},">>, 76}, {<<"[{ns_bootstrap,start,0,[{file,\"src/ns_bootstrap.erl\"},{line,31}]},">>, 66}, {<<" {ns_server,start,[normal,[]]}}}}">>, 46}, {<<" [{file,\"proc_lib.erl\"},{line,249}]}]}}}}}}}}},">>, 71}, {<<" {proc_lib,init_p_do_apply,3,">>, 52}, {<<" [{file,\"gen_statem.erl\"},{line,1120}]},">>, 64}, {<<" {gen_statem,loop_state_callback,11,">>, 59}, {<<" [{file,\"src/chronicle_rsm.erl\"},{line,1020}]},">>, 71}, {<<" {chronicle_rsm,handle_query,4,">>, 54}, {<<" [{file,\"src/chronicle_kv.erl\"},{line,668}]},">>, 69}, {<<" {chronicle_kv,handle_prepare_txn,4,">>, 59}, {<<" [{file,\"src/chronicle_kv.erl\"},{line,968}]},">>, 69}, {<<" {chronicle_kv,txn_with_conditions,2,">>, 60}, {<<" {lists,foldl,3,[{file,\"lists.erl\"},{line,1263}]},">>, 73}], [{<<" [{file,\"src/chronicle_compat.erl\"},{line,240}]},">>, 73}]}}, undefined,undefined,[],0} ** Reason for termination == ** {abnormal,3} [error_logger:error,2021-06-03T09:43:22.310Z,babysitter_of_ns_1@cb.local:<0.185.0>:ale_error_logger_handler:do_log:101] =========================CRASH REPORT========================= crasher: initial call: ns_port_server:init/1 pid: <0.185.0> registered_name: [] exception exit: {abnormal,3} in function gen_server:handle_common_reply/8 (gen_server.erl, line 751) ancestors: [<0.184.0>,<0.183.0>,child_ns_server_sup,ns_babysitter_sup, <0.107.0>] message_queue_len: 1 messages: [{#Port<0.32>,eof}] links: [<0.184.0>,#Port<0.32>] dictionary: [] trap_exit: true status: running heap_size: 10958 stack_size: 27 reductions: 78519 neighbours: [ns_server:info,2021-06-03T09:43:22.310Z,babysitter_of_ns_1@cb.local:<0.184.0>:supervisor_cushion:handle_info:54]Cushion managed supervisor for ns_server failed: {abnormal,3} [error_logger:error,2021-06-03T09:43:22.310Z,babysitter_of_ns_1@cb.local:<0.184.0>:ale_error_logger_handler:do_log:101] =========================ERROR REPORT========================= ** Generic server <0.184.0> terminating ** Last message in was {die,{abnormal,3}} ** When Server state == {state,ns_server,5000,-576460657957274795,undefined, infinity} ** Reason for termination == ** {abnormal,3} [error_logger:error,2021-06-03T09:43:22.310Z,babysitter_of_ns_1@cb.local:<0.184.0>:ale_error_logger_handler:do_log:101] =========================CRASH REPORT========================= crasher: initial call: supervisor_cushion:init/1 pid: <0.184.0> registered_name: [] exception exit: {abnormal,3} in function gen_server:handle_common_reply/8 (gen_server.erl, line 751) ancestors: [<0.183.0>,child_ns_server_sup,ns_babysitter_sup,<0.107.0>] message_queue_len: 0 messages: [] links: [<0.183.0>] dictionary: [] trap_exit: true status: running heap_size: 2586 stack_size: 27 reductions: 8831 neighbours: [error_logger:error,2021-06-03T09:43:22.311Z,babysitter_of_ns_1@cb.local:<0.183.0>:ale_error_logger_handler:do_log:101] =========================CRASH REPORT========================= crasher: initial call: erlang:apply/2 pid: <0.183.0> registered_name: [] exception exit: {abnormal,3} in function restartable:loop/4 (src/restartable.erl, line 63) ancestors: [child_ns_server_sup,ns_babysitter_sup,<0.107.0>] message_queue_len: 0 messages: [] links: [<0.120.0>] dictionary: [] trap_exit: true status: running heap_size: 10958 stack_size: 27 reductions: 3120 neighbours: [error_logger:error,2021-06-03T09:43:22.311Z,babysitter_of_ns_1@cb.local:child_ns_server_sup<0.120.0>:ale_error_logger_handler:do_log:101] =========================SUPERVISOR REPORT========================= supervisor: {local,child_ns_server_sup} errorContext: child_terminated reason: {abnormal,3} offender: [{pid,<0.183.0>}, {id,{ns_server,"/opt/couchbase/lib/erlang/bin/erl", ["+A","16","-smp","enable","+sbt","u","+P","327680", "+K","true","+swt","low","+sbwt","none","+MMmcs","30", "+e102400","-setcookie","nocookie","-kernel","logger", "[{handler, default, undefined}]","-user","user_io", "-run","child_erlang","child_start","ns_bootstrap", "--","-smp","enable","-kernel","logger", "[{handler, default, undefined}]","inetrc", "\"/opt/couchbase/etc/couchbase/hosts.cfg\"", "dist_config_file", "\"/opt/couchbase/var/lib/couchbase/config/dist_cfg\"", "-proto_dist","cb","-epmd_module","cb_epmd", "-ssl_dist_optfile", "/opt/couchbase/etc/couchbase/ssl_dist_opts", "-couch_ini","/opt/couchbase/etc/couchdb/default.ini", "/opt/couchbase/etc/couchdb/default.d/capi.ini", "/opt/couchbase/etc/couchdb/default.d/geocouch.ini", "/opt/couchbase/etc/couchdb/local.ini"], [{env, [{"NS_SERVER_BABYSITTER_PID","4690"}, {"CHILD_ERLANG_ENV_ARGS", "[{ns_babysitter,\n [{cookiefile,\n \"/opt/couchbase/var/lib/couchbase/couchbase-server.babysitter.cookie\"},\n {pidfile,\"/opt/couchbase/var/lib/couchbase/couchbase-server.pid\"},\n {nodefile,\n \"/opt/couchbase/var/lib/couchbase/couchbase-server.babysitter.node\"}]},\n {ns_server,\n [{loglevel_ns_doctor,debug},\n {config_path,\"/opt/couchbase/etc/couchbase/static_config\"},\n {max_r,20},\n {loglevel_xdcr,debug},\n {error_logger_mf_dir,\"/opt/couchbase/var/lib/couchbase/logs\"},\n {loglevel_menelaus,debug},\n {nodefile,\"/opt/couchbase/var/lib/couchbase/couchbase-server.node\"},\n {loglevel_ns_server,debug},\n {disk_sink_opts,\n [{rotation,\n [{compress,true},\n {size,41943040},\n {num_files,10},\n {buffer_size_max,52428800}]}]},\n {loglevel_cluster,debug},\n {max_t,10},\n {loglevel_rebalance,debug},\n {path_config_libdir,\"/opt/couchbase/lib\"},\n {net_kernel_verbosity,10},\n {loglevel_mapreduce_errors,debug},\n {loglevel_views,debug},\n {loglevel_default,debug},\n {path_config_bindir,\"/opt/couchbase/bin\"},\n {path_config_tmpdir,\"/opt/couchbase/var/lib/couchbase/tmp\"},\n {loglevel_stats,debug},\n {loglevel_user,debug},\n {disk_sink_opts_json_rpc,\n [{rotation,\n [{compress,true},\n {size,41943040},\n {num_files,2},\n {buffer_size_max,52428800}]}]},\n {loglevel_couchdb,info},\n {path_config_secdir,\"/opt/couchbase/etc/security\"},\n {loglevel_access,info},\n {path_config_etcdir,\"/opt/couchbase/etc/couchbase\"},\n {path_config_datadir,\"/opt/couchbase/var/lib/couchbase\"},\n {loglevel_cbas,debug},\n {loglevel_error_logger,debug}]}]"}, {"ERL_CRASH_DUMP", "erl_crash.dump.1622713300.4690.ns_server"}]}, exit_status,use_stdio,stream,eof]}}, {mfargs, {restartable,start_link, [{supervisor_cushion,start_link, [ns_server,5000,infinity,ns_port_server, start_link, [#Fun]]}, 86400000]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,worker}] [ns_server:debug,2021-06-03T09:43:22.312Z,babysitter_of_ns_1@cb.local:<0.189.0>:supervisor_cushion:init:33]starting ns_port_server with delay of 5000 [ns_server:debug,2021-06-03T09:43:22.317Z,babysitter_of_ns_1@cb.local:<0.188.0>:restartable:start_child:92]Started child process <0.189.0> MFA: {supervisor_cushion,start_link, [ns_server,5000,infinity,ns_port_server,start_link, [#Fun]]} [error_logger:info,2021-06-03T09:43:22.318Z,babysitter_of_ns_1@cb.local:child_ns_server_sup<0.120.0>:ale_error_logger_handler:do_log:101] =========================PROGRESS REPORT========================= supervisor: {local,child_ns_server_sup} started: [{pid,<0.188.0>}, {id,{ns_server,"/opt/couchbase/lib/erlang/bin/erl", ["+A","16","-smp","enable","+sbt","u","+P","327680", "+K","true","+swt","low","+sbwt","none","+MMmcs","30", "+e102400","-setcookie","nocookie","-kernel","logger", "[{handler, default, undefined}]","-user","user_io", "-run","child_erlang","child_start","ns_bootstrap", "--","-smp","enable","-kernel","logger", "[{handler, default, undefined}]","inetrc", "\"/opt/couchbase/etc/couchbase/hosts.cfg\"", "dist_config_file", "\"/opt/couchbase/var/lib/couchbase/config/dist_cfg\"", "-proto_dist","cb","-epmd_module","cb_epmd", "-ssl_dist_optfile", "/opt/couchbase/etc/couchbase/ssl_dist_opts", "-couch_ini","/opt/couchbase/etc/couchdb/default.ini", "/opt/couchbase/etc/couchdb/default.d/capi.ini", "/opt/couchbase/etc/couchdb/default.d/geocouch.ini", "/opt/couchbase/etc/couchdb/local.ini"], [{env, [{"NS_SERVER_BABYSITTER_PID","4690"}, {"CHILD_ERLANG_ENV_ARGS", "[{ns_babysitter,\n [{cookiefile,\n \"/opt/couchbase/var/lib/couchbase/couchbase-server.babysitter.cookie\"},\n {pidfile,\"/opt/couchbase/var/lib/couchbase/couchbase-server.pid\"},\n {nodefile,\n \"/opt/couchbase/var/lib/couchbase/couchbase-server.babysitter.node\"}]},\n {ns_server,\n [{loglevel_ns_doctor,debug},\n {config_path,\"/opt/couchbase/etc/couchbase/static_config\"},\n {max_r,20},\n {loglevel_xdcr,debug},\n {error_logger_mf_dir,\"/opt/couchbase/var/lib/couchbase/logs\"},\n {loglevel_menelaus,debug},\n {nodefile,\"/opt/couchbase/var/lib/couchbase/couchbase-server.node\"},\n {loglevel_ns_server,debug},\n {disk_sink_opts,\n [{rotation,\n [{compress,true},\n {size,41943040},\n {num_files,10},\n {buffer_size_max,52428800}]}]},\n {loglevel_cluster,debug},\n {max_t,10},\n {loglevel_rebalance,debug},\n {path_config_libdir,\"/opt/couchbase/lib\"},\n {net_kernel_verbosity,10},\n {loglevel_mapreduce_errors,debug},\n {loglevel_views,debug},\n {loglevel_default,debug},\n {path_config_bindir,\"/opt/couchbase/bin\"},\n {path_config_tmpdir,\"/opt/couchbase/var/lib/couchbase/tmp\"},\n {loglevel_stats,debug},\n {loglevel_user,debug},\n {disk_sink_opts_json_rpc,\n [{rotation,\n [{compress,true},\n {size,41943040},\n {num_files,2},\n {buffer_size_max,52428800}]}]},\n {loglevel_couchdb,info},\n {path_config_secdir,\"/opt/couchbase/etc/security\"},\n {loglevel_access,info},\n {path_config_etcdir,\"/opt/couchbase/etc/couchbase\"},\n {path_config_datadir,\"/opt/couchbase/var/lib/couchbase\"},\n {loglevel_cbas,debug},\n {loglevel_error_logger,debug}]}]"}, {"ERL_CRASH_DUMP", "erl_crash.dump.1622713300.4690.ns_server"}]}, exit_status,use_stdio,stream,eof]}}, {mfargs, {restartable,start_link, [{supervisor_cushion,start_link, [ns_server,5000,infinity,ns_port_server,start_link, [#Fun]]}, 86400000]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,worker}] [ns_server:debug,2021-06-03T09:43:24.269Z,babysitter_of_ns_1@cb.local:cb_dist<0.65.0>:cb_dist:info_msg:790]cb_dist: Accepted new connection from <0.69.0> DistCtrl #Port<0.35>: {con, #Ref<0.1855568256.1190920193.231448>, inet_tcp_dist, undefined, undefined} [ns_server:debug,2021-06-03T09:43:24.269Z,babysitter_of_ns_1@cb.local:net_kernel<0.68.0>:cb_dist:info_msg:790]cb_dist: Accepting connection from acceptor <0.69.0> using module inet_tcp_dist [ns_server:debug,2021-06-03T09:43:24.269Z,babysitter_of_ns_1@cb.local:cb_dist<0.65.0>:cb_dist:info_msg:790]cb_dist: Updated connection: {con,#Ref<0.1855568256.1190920193.231448>, inet_tcp_dist,<0.191.0>, #Ref<0.1855568256.1190920193.231451>} [ns_server:info,2021-06-03T09:43:26.929Z,babysitter_of_ns_1@cb.local:<0.190.0>:ns_port_server:log:221]ns_server<0.190.0>: working as port ns_server<0.190.0>: [os_mon] memory supervisor port (memsup): Erlang has closed ns_server<0.190.0>: [os_mon] cpu supervisor port (cpu_sup): Erlang has closed [error_logger:info,2021-06-03T09:43:26.996Z,babysitter_of_ns_1@cb.local:net_kernel<0.68.0>:ale_error_logger_handler:do_log:101] =========================NOTICE REPORT========================= {net_kernel,{'EXIT',#Port<0.35>,normal}} [error_logger:info,2021-06-03T09:43:26.996Z,babysitter_of_ns_1@cb.local:net_kernel<0.68.0>:ale_error_logger_handler:do_log:101] =========================NOTICE REPORT========================= {net_kernel,{'EXIT',<0.191.0>,connection_closed}} [ns_server:debug,2021-06-03T09:43:26.996Z,babysitter_of_ns_1@cb.local:cb_dist<0.65.0>:cb_dist:info_msg:790]cb_dist: Connection down: {con,#Ref<0.1855568256.1190920193.231448>, inet_tcp_dist,<0.191.0>, #Ref<0.1855568256.1190920193.231451>} [ns_server:info,2021-06-03T09:43:27.584Z,babysitter_of_ns_1@cb.local:<0.190.0>:ns_port_server:log:221]ns_server<0.190.0>: Crap error:{badmatch, ns_server<0.190.0>: {error, ns_server<0.190.0>: {{shutdown, ns_server<0.190.0>: {failed_to_start_child,ns_server_cluster_sup, ns_server<0.190.0>: {shutdown, ns_server<0.190.0>: {failed_to_start_child,ns_server_nodes_sup, ns_server<0.190.0>: {shutdown, ns_server<0.190.0>: {failed_to_start_child,ns_server_sup, ns_server<0.190.0>: {shutdown, ns_server<0.190.0>: {failed_to_start_child,memcached_passwords, ns_server<0.190.0>: {{badmatch,{error,not_found}}, ns_server<0.190.0>: [{ns_bucket,fetch_snapshot,2, ns_server<0.190.0>: [{file,"src/ns_bucket.erl"},{line,168}]}, ns_server<0.190.0>: {chronicle_compat, ns_server<0.190.0>: '-get_snapshot_with_revision/2-fun-0-',3, ns_server<0.190.0>: [{file,"src/chronicle_compat.erl"},{line,240}]}, ns_server<0.190.0>: {lists,foldl,3,[{file,"lists.erl"},{line,1263}]}, ns_server<0.190.0>: {chronicle_kv,txn_with_conditions,2, ns_server<0.190.0>: [{file,"src/chronicle_kv.erl"},{line,968}]}, ns_server<0.190.0>: {chronicle_kv,handle_prepare_txn,4, ns_server<0.190.0>: [{file,"src/chronicle_kv.erl"},{line,668}]}, ns_server<0.190.0>: {chronicle_rsm,handle_query,4, ns_server<0.190.0>: [{file,"src/chronicle_rsm.erl"},{line,1020}]}, ns_server<0.190.0>: {gen_statem,loop_state_callback,11, ns_server<0.190.0>: [{file,"gen_statem.erl"},{line,1120}]}, ns_server<0.190.0>: {proc_lib,init_p_do_apply,3, ns_server<0.190.0>: [{file,"proc_lib.erl"},{line,249}]}]}}}}}}}}}, ns_server<0.190.0>: {ns_server,start,[normal,[]]}}}} ns_server<0.190.0>: [{ns_bootstrap,start,0,[{file,"src/ns_bootstrap.erl"},{line,31}]}, ns_server<0.190.0>: {child_erlang,do_child_start,1,[{file,"src/child_erlang.erl"},{line,105}]}, ns_server<0.190.0>: {child_erlang,child_start,1,[{file,"src/child_erlang.erl"},{line,83}]}, ns_server<0.190.0>: {init,start_em,1,[]}, ns_server<0.190.0>: {init,do_boot,3,[]}] [error_logger:error,2021-06-03T09:43:28.389Z,babysitter_of_ns_1@cb.local:<0.190.0>:ale_error_logger_handler:do_log:101] =========================ERROR REPORT========================= ** Generic server <0.190.0> terminating ** Last message in was {#Port<0.34>,{exit_status,3}} ** When Server state == {state,#Port<0.34>, {ns_server,"/opt/couchbase/lib/erlang/bin/erl", ["+A","16","-smp","enable","+sbt","u","+P", "327680","+K","true","+swt","low","+sbwt", "none","+MMmcs","30","+e102400","-setcookie", "nocookie","-kernel","logger", "[{handler, default, undefined}]","-user", "user_io","-run","child_erlang", "child_start","ns_bootstrap","--","-smp", "enable","-kernel","logger", "[{handler, default, undefined}]","inetrc", "\"/opt/couchbase/etc/couchbase/hosts.cfg\"", "dist_config_file", "\"/opt/couchbase/var/lib/couchbase/config/dist_cfg\"", "-proto_dist","cb","-epmd_module","cb_epmd", "-ssl_dist_optfile", "/opt/couchbase/etc/couchbase/ssl_dist_opts", "-couch_ini", "/opt/couchbase/etc/couchdb/default.ini", "/opt/couchbase/etc/couchdb/default.d/capi.ini", "/opt/couchbase/etc/couchdb/default.d/geocouch.ini", "/opt/couchbase/etc/couchdb/local.ini"], [{env, [{"NS_SERVER_BABYSITTER_PID","4690"}, {"CHILD_ERLANG_ENV_ARGS", "[{ns_babysitter,\n [{cookiefile,\n \"/opt/couchbase/var/lib/couchbase/couchbase-server.babysitter.cookie\"},\n {pidfile,\"/opt/couchbase/var/lib/couchbase/couchbase-server.pid\"},\n {nodefile,\n \"/opt/couchbase/var/lib/couchbase/couchbase-server.babysitter.node\"}]},\n {ns_server,\n [{loglevel_ns_doctor,debug},\n {config_path,\"/opt/couchbase/etc/couchbase/static_config\"},\n {max_r,20},\n {loglevel_xdcr,debug},\n {error_logger_mf_dir,\"/opt/couchbase/var/lib/couchbase/logs\"},\n {loglevel_menelaus,debug},\n {nodefile,\"/opt/couchbase/var/lib/couchbase/couchbase-server.node\"},\n {loglevel_ns_server,debug},\n {disk_sink_opts,\n [{rotation,\n [{compress,true},\n {size,41943040},\n {num_files,10},\n {buffer_size_max,52428800}]}]},\n {loglevel_cluster,debug},\n {max_t,10},\n {loglevel_rebalance,debug},\n {path_config_libdir,\"/opt/couchbase/lib\"},\n {net_kernel_verbosity,10},\n {loglevel_mapreduce_errors,debug},\n {loglevel_views,debug},\n {loglevel_default,debug},\n {path_config_bindir,\"/opt/couchbase/bin\"},\n {path_config_tmpdir,\"/opt/couchbase/var/lib/couchbase/tmp\"},\n {loglevel_stats,debug},\n {loglevel_user,debug},\n {disk_sink_opts_json_rpc,\n [{rotation,\n [{compress,true},\n {size,41943040},\n {num_files,2},\n {buffer_size_max,52428800}]}]},\n {loglevel_couchdb,info},\n {path_config_secdir,\"/opt/couchbase/etc/security\"},\n {loglevel_access,info},\n {path_config_etcdir,\"/opt/couchbase/etc/couchbase\"},\n {path_config_datadir,\"/opt/couchbase/var/lib/couchbase\"},\n {loglevel_cbas,debug},\n {loglevel_error_logger,debug}]}]"}, {"ERL_CRASH_DUMP", "erl_crash.dump.1622713300.4690.ns_server"}]}, exit_status,use_stdio,stream,eof]}, {ringbuffer,1077,1024, {[{<<" {init,do_boot,3,[]}]">>,21}, {<<" {init,start_em,1,[]},">>,22}, {<<" {child_erlang,child_start,1,[{file,\"src/child_erlang.erl\"},{line,83}]},">>, 72}, {<<" {child_erlang,do_child_start,1,[{file,\"src/child_erlang.erl\"},{line,105}]},">>, 76}, {<<"[{ns_bootstrap,start,0,[{file,\"src/ns_bootstrap.erl\"},{line,31}]},">>, 66}, {<<" {ns_server,start,[normal,[]]}}}}">>, 46}, {<<" [{file,\"proc_lib.erl\"},{line,249}]}]}}}}}}}}},">>, 71}, {<<" {proc_lib,init_p_do_apply,3,">>, 52}, {<<" [{file,\"gen_statem.erl\"},{line,1120}]},">>, 64}, {<<" {gen_statem,loop_state_callback,11,">>, 59}, {<<" [{file,\"src/chronicle_rsm.erl\"},{line,1020}]},">>, 71}, {<<" {chronicle_rsm,handle_query,4,">>, 54}, {<<" [{file,\"src/chronicle_kv.erl\"},{line,668}]},">>, 69}, {<<" {chronicle_kv,handle_prepare_txn,4,">>, 59}, {<<" [{file,\"src/chronicle_kv.erl\"},{line,968}]},">>, 69}, {<<" {chronicle_kv,txn_with_conditions,2,">>, 60}, {<<" {lists,foldl,3,[{file,\"lists.erl\"},{line,1263}]},">>, 73}], [{<<" [{file,\"src/chronicle_compat.erl\"},{line,240}]},">>, 73}]}}, undefined,undefined,[],0} ** Reason for termination == ** {abnormal,3} [error_logger:error,2021-06-03T09:43:28.394Z,babysitter_of_ns_1@cb.local:<0.190.0>:ale_error_logger_handler:do_log:101] =========================CRASH REPORT========================= crasher: initial call: ns_port_server:init/1 pid: <0.190.0> registered_name: [] exception exit: {abnormal,3} in function gen_server:handle_common_reply/8 (gen_server.erl, line 751) ancestors: [<0.189.0>,<0.188.0>,child_ns_server_sup,ns_babysitter_sup, <0.107.0>] message_queue_len: 1 messages: [{#Port<0.34>,eof}] links: [<0.189.0>,#Port<0.34>] dictionary: [] trap_exit: true status: running heap_size: 10958 stack_size: 27 reductions: 78519 neighbours: [ns_server:info,2021-06-03T09:43:28.395Z,babysitter_of_ns_1@cb.local:<0.189.0>:supervisor_cushion:handle_info:54]Cushion managed supervisor for ns_server failed: {abnormal,3} [error_logger:error,2021-06-03T09:43:28.395Z,babysitter_of_ns_1@cb.local:<0.189.0>:ale_error_logger_handler:do_log:101] =========================ERROR REPORT========================= ** Generic server <0.189.0> terminating ** Last message in was {die,{abnormal,3}} ** When Server state == {state,ns_server,5000,-576460651847097507,undefined, infinity} ** Reason for termination == ** {abnormal,3} [error_logger:error,2021-06-03T09:43:28.395Z,babysitter_of_ns_1@cb.local:<0.189.0>:ale_error_logger_handler:do_log:101] =========================CRASH REPORT========================= crasher: initial call: supervisor_cushion:init/1 pid: <0.189.0> registered_name: [] exception exit: {abnormal,3} in function gen_server:handle_common_reply/8 (gen_server.erl, line 751) ancestors: [<0.188.0>,child_ns_server_sup,ns_babysitter_sup,<0.107.0>] message_queue_len: 0 messages: [] links: [<0.188.0>] dictionary: [] trap_exit: true status: running heap_size: 2586 stack_size: 27 reductions: 8831 neighbours: [error_logger:error,2021-06-03T09:43:28.395Z,babysitter_of_ns_1@cb.local:<0.188.0>:ale_error_logger_handler:do_log:101] =========================CRASH REPORT========================= crasher: initial call: erlang:apply/2 pid: <0.188.0> registered_name: [] exception exit: {abnormal,3} in function restartable:loop/4 (src/restartable.erl, line 63) ancestors: [child_ns_server_sup,ns_babysitter_sup,<0.107.0>] message_queue_len: 0 messages: [] links: [<0.120.0>] dictionary: [] trap_exit: true status: running heap_size: 10958 stack_size: 27 reductions: 3120 neighbours: [error_logger:error,2021-06-03T09:43:28.395Z,babysitter_of_ns_1@cb.local:child_ns_server_sup<0.120.0>:ale_error_logger_handler:do_log:101] =========================SUPERVISOR REPORT========================= supervisor: {local,child_ns_server_sup} errorContext: child_terminated reason: {abnormal,3} offender: [{pid,<0.188.0>}, {id,{ns_server,"/opt/couchbase/lib/erlang/bin/erl", ["+A","16","-smp","enable","+sbt","u","+P","327680", "+K","true","+swt","low","+sbwt","none","+MMmcs","30", "+e102400","-setcookie","nocookie","-kernel","logger", "[{handler, default, undefined}]","-user","user_io", "-run","child_erlang","child_start","ns_bootstrap", "--","-smp","enable","-kernel","logger", "[{handler, default, undefined}]","inetrc", "\"/opt/couchbase/etc/couchbase/hosts.cfg\"", "dist_config_file", "\"/opt/couchbase/var/lib/couchbase/config/dist_cfg\"", "-proto_dist","cb","-epmd_module","cb_epmd", "-ssl_dist_optfile", "/opt/couchbase/etc/couchbase/ssl_dist_opts", "-couch_ini","/opt/couchbase/etc/couchdb/default.ini", "/opt/couchbase/etc/couchdb/default.d/capi.ini", "/opt/couchbase/etc/couchdb/default.d/geocouch.ini", "/opt/couchbase/etc/couchdb/local.ini"], [{env, [{"NS_SERVER_BABYSITTER_PID","4690"}, {"CHILD_ERLANG_ENV_ARGS", "[{ns_babysitter,\n [{cookiefile,\n \"/opt/couchbase/var/lib/couchbase/couchbase-server.babysitter.cookie\"},\n {pidfile,\"/opt/couchbase/var/lib/couchbase/couchbase-server.pid\"},\n {nodefile,\n \"/opt/couchbase/var/lib/couchbase/couchbase-server.babysitter.node\"}]},\n {ns_server,\n [{loglevel_ns_doctor,debug},\n {config_path,\"/opt/couchbase/etc/couchbase/static_config\"},\n {max_r,20},\n {loglevel_xdcr,debug},\n {error_logger_mf_dir,\"/opt/couchbase/var/lib/couchbase/logs\"},\n {loglevel_menelaus,debug},\n {nodefile,\"/opt/couchbase/var/lib/couchbase/couchbase-server.node\"},\n {loglevel_ns_server,debug},\n {disk_sink_opts,\n [{rotation,\n [{compress,true},\n {size,41943040},\n {num_files,10},\n {buffer_size_max,52428800}]}]},\n {loglevel_cluster,debug},\n {max_t,10},\n {loglevel_rebalance,debug},\n {path_config_libdir,\"/opt/couchbase/lib\"},\n {net_kernel_verbosity,10},\n {loglevel_mapreduce_errors,debug},\n {loglevel_views,debug},\n {loglevel_default,debug},\n {path_config_bindir,\"/opt/couchbase/bin\"},\n {path_config_tmpdir,\"/opt/couchbase/var/lib/couchbase/tmp\"},\n {loglevel_stats,debug},\n {loglevel_user,debug},\n {disk_sink_opts_json_rpc,\n [{rotation,\n [{compress,true},\n {size,41943040},\n {num_files,2},\n {buffer_size_max,52428800}]}]},\n {loglevel_couchdb,info},\n {path_config_secdir,\"/opt/couchbase/etc/security\"},\n {loglevel_access,info},\n {path_config_etcdir,\"/opt/couchbase/etc/couchbase\"},\n {path_config_datadir,\"/opt/couchbase/var/lib/couchbase\"},\n {loglevel_cbas,debug},\n {loglevel_error_logger,debug}]}]"}, {"ERL_CRASH_DUMP", "erl_crash.dump.1622713300.4690.ns_server"}]}, exit_status,use_stdio,stream,eof]}}, {mfargs, {restartable,start_link, [{supervisor_cushion,start_link, [ns_server,5000,infinity,ns_port_server, start_link, [#Fun]]}, 86400000]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,worker}] [ns_server:debug,2021-06-03T09:43:28.396Z,babysitter_of_ns_1@cb.local:<0.194.0>:supervisor_cushion:init:33]starting ns_port_server with delay of 5000 [ns_server:debug,2021-06-03T09:43:28.402Z,babysitter_of_ns_1@cb.local:<0.193.0>:restartable:start_child:92]Started child process <0.194.0> MFA: {supervisor_cushion,start_link, [ns_server,5000,infinity,ns_port_server,start_link, [#Fun]]} [error_logger:info,2021-06-03T09:43:28.402Z,babysitter_of_ns_1@cb.local:child_ns_server_sup<0.120.0>:ale_error_logger_handler:do_log:101] =========================PROGRESS REPORT========================= supervisor: {local,child_ns_server_sup} started: [{pid,<0.193.0>}, {id,{ns_server,"/opt/couchbase/lib/erlang/bin/erl", ["+A","16","-smp","enable","+sbt","u","+P","327680", "+K","true","+swt","low","+sbwt","none","+MMmcs","30", "+e102400","-setcookie","nocookie","-kernel","logger", "[{handler, default, undefined}]","-user","user_io", "-run","child_erlang","child_start","ns_bootstrap", "--","-smp","enable","-kernel","logger", "[{handler, default, undefined}]","inetrc", "\"/opt/couchbase/etc/couchbase/hosts.cfg\"", "dist_config_file", "\"/opt/couchbase/var/lib/couchbase/config/dist_cfg\"", "-proto_dist","cb","-epmd_module","cb_epmd", "-ssl_dist_optfile", "/opt/couchbase/etc/couchbase/ssl_dist_opts", "-couch_ini","/opt/couchbase/etc/couchdb/default.ini", "/opt/couchbase/etc/couchdb/default.d/capi.ini", "/opt/couchbase/etc/couchdb/default.d/geocouch.ini", "/opt/couchbase/etc/couchdb/local.ini"], [{env, [{"NS_SERVER_BABYSITTER_PID","4690"}, {"CHILD_ERLANG_ENV_ARGS", "[{ns_babysitter,\n [{cookiefile,\n \"/opt/couchbase/var/lib/couchbase/couchbase-server.babysitter.cookie\"},\n {pidfile,\"/opt/couchbase/var/lib/couchbase/couchbase-server.pid\"},\n {nodefile,\n \"/opt/couchbase/var/lib/couchbase/couchbase-server.babysitter.node\"}]},\n {ns_server,\n [{loglevel_ns_doctor,debug},\n {config_path,\"/opt/couchbase/etc/couchbase/static_config\"},\n {max_r,20},\n {loglevel_xdcr,debug},\n {error_logger_mf_dir,\"/opt/couchbase/var/lib/couchbase/logs\"},\n {loglevel_menelaus,debug},\n {nodefile,\"/opt/couchbase/var/lib/couchbase/couchbase-server.node\"},\n {loglevel_ns_server,debug},\n {disk_sink_opts,\n [{rotation,\n [{compress,true},\n {size,41943040},\n {num_files,10},\n {buffer_size_max,52428800}]}]},\n {loglevel_cluster,debug},\n {max_t,10},\n {loglevel_rebalance,debug},\n {path_config_libdir,\"/opt/couchbase/lib\"},\n {net_kernel_verbosity,10},\n {loglevel_mapreduce_errors,debug},\n {loglevel_views,debug},\n {loglevel_default,debug},\n {path_config_bindir,\"/opt/couchbase/bin\"},\n {path_config_tmpdir,\"/opt/couchbase/var/lib/couchbase/tmp\"},\n {loglevel_stats,debug},\n {loglevel_user,debug},\n {disk_sink_opts_json_rpc,\n [{rotation,\n [{compress,true},\n {size,41943040},\n {num_files,2},\n {buffer_size_max,52428800}]}]},\n {loglevel_couchdb,info},\n {path_config_secdir,\"/opt/couchbase/etc/security\"},\n {loglevel_access,info},\n {path_config_etcdir,\"/opt/couchbase/etc/couchbase\"},\n {path_config_datadir,\"/opt/couchbase/var/lib/couchbase\"},\n {loglevel_cbas,debug},\n {loglevel_error_logger,debug}]}]"}, {"ERL_CRASH_DUMP", "erl_crash.dump.1622713300.4690.ns_server"}]}, exit_status,use_stdio,stream,eof]}}, {mfargs, {restartable,start_link, [{supervisor_cushion,start_link, [ns_server,5000,infinity,ns_port_server,start_link, [#Fun]]}, 86400000]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,worker}] [ns_server:debug,2021-06-03T09:43:30.358Z,babysitter_of_ns_1@cb.local:cb_dist<0.65.0>:cb_dist:info_msg:790]cb_dist: Accepted new connection from <0.69.0> DistCtrl #Port<0.37>: {con, #Ref<0.1855568256.1190920193.231487>, inet_tcp_dist, undefined, undefined} [ns_server:debug,2021-06-03T09:43:30.358Z,babysitter_of_ns_1@cb.local:net_kernel<0.68.0>:cb_dist:info_msg:790]cb_dist: Accepting connection from acceptor <0.69.0> using module inet_tcp_dist [ns_server:debug,2021-06-03T09:43:30.358Z,babysitter_of_ns_1@cb.local:cb_dist<0.65.0>:cb_dist:info_msg:790]cb_dist: Updated connection: {con,#Ref<0.1855568256.1190920193.231487>, inet_tcp_dist,<0.196.0>, #Ref<0.1855568256.1190920193.231490>} [ns_server:info,2021-06-03T09:43:32.922Z,babysitter_of_ns_1@cb.local:<0.195.0>:ns_port_server:log:221]ns_server<0.195.0>: working as port ns_server<0.195.0>: [os_mon] memory supervisor port (memsup): Erlang has closed ns_server<0.195.0>: [os_mon] cpu supervisor port (cpu_sup): Erlang has closed [error_logger:info,2021-06-03T09:43:32.982Z,babysitter_of_ns_1@cb.local:net_kernel<0.68.0>:ale_error_logger_handler:do_log:101] =========================NOTICE REPORT========================= {net_kernel,{'EXIT',#Port<0.37>,normal}} [error_logger:info,2021-06-03T09:43:32.982Z,babysitter_of_ns_1@cb.local:net_kernel<0.68.0>:ale_error_logger_handler:do_log:101] =========================NOTICE REPORT========================= {net_kernel,{'EXIT',<0.196.0>,connection_closed}} [ns_server:debug,2021-06-03T09:43:32.983Z,babysitter_of_ns_1@cb.local:cb_dist<0.65.0>:cb_dist:info_msg:790]cb_dist: Connection down: {con,#Ref<0.1855568256.1190920193.231487>, inet_tcp_dist,<0.196.0>, #Ref<0.1855568256.1190920193.231490>} [ns_server:info,2021-06-03T09:43:33.569Z,babysitter_of_ns_1@cb.local:<0.195.0>:ns_port_server:log:221]ns_server<0.195.0>: Crap error:{badmatch, ns_server<0.195.0>: {error, ns_server<0.195.0>: {{shutdown, ns_server<0.195.0>: {failed_to_start_child,ns_server_cluster_sup, ns_server<0.195.0>: {shutdown, ns_server<0.195.0>: {failed_to_start_child,ns_server_nodes_sup, ns_server<0.195.0>: {shutdown, ns_server<0.195.0>: {failed_to_start_child,ns_server_sup, ns_server<0.195.0>: {shutdown, ns_server<0.195.0>: {failed_to_start_child,memcached_passwords, ns_server<0.195.0>: {{badmatch,{error,not_found}}, ns_server<0.195.0>: [{ns_bucket,fetch_snapshot,2, ns_server<0.195.0>: [{file,"src/ns_bucket.erl"},{line,168}]}, ns_server<0.195.0>: {chronicle_compat, ns_server<0.195.0>: '-get_snapshot_with_revision/2-fun-0-',3, ns_server<0.195.0>: [{file,"src/chronicle_compat.erl"},{line,240}]}, ns_server<0.195.0>: {lists,foldl,3,[{file,"lists.erl"},{line,1263}]}, ns_server<0.195.0>: {chronicle_kv,txn_with_conditions,2, ns_server<0.195.0>: [{file,"src/chronicle_kv.erl"},{line,968}]}, ns_server<0.195.0>: {chronicle_kv,handle_prepare_txn,4, ns_server<0.195.0>: [{file,"src/chronicle_kv.erl"},{line,668}]}, ns_server<0.195.0>: {chronicle_rsm,handle_query,4, ns_server<0.195.0>: [{file,"src/chronicle_rsm.erl"},{line,1020}]}, ns_server<0.195.0>: {gen_statem,loop_state_callback,11, ns_server<0.195.0>: [{file,"gen_statem.erl"},{line,1120}]}, ns_server<0.195.0>: {proc_lib,init_p_do_apply,3, ns_server<0.195.0>: [{file,"proc_lib.erl"},{line,249}]}]}}}}}}}}}, ns_server<0.195.0>: {ns_server,start,[normal,[]]}}}} ns_server<0.195.0>: [{ns_bootstrap,start,0,[{file,"src/ns_bootstrap.erl"},{line,31}]}, ns_server<0.195.0>: {child_erlang,do_child_start,1,[{file,"src/child_erlang.erl"},{line,105}]}, ns_server<0.195.0>: {child_erlang,child_start,1,[{file,"src/child_erlang.erl"},{line,83}]}, ns_server<0.195.0>: {init,start_em,1,[]}, ns_server<0.195.0>: {init,do_boot,3,[]}] {noformat}