Crash reports in babysitter.log - crash_report #{label=>{proc_lib,crash},report=>[[{initial_call,{ns_memcached,init,['Argument__1']}},{pid,<0.25327.1>},{registered_name,'ns_memcached-bucket8'},

Starting rebalance, KeepNodes = ['ns_1@172.23.104.137','ns_1@172.23.104.155',

'ns_1@172.23.104.157','ns_1@172.23.104.5',

'ns_1@172.23.104.67','ns_1@172.23.104.69',

'ns_1@172.23.104.70','ns_1@172.23.105.107',

'ns_1@172.23.105.111','ns_1@172.23.106.188',

'ns_1@172.23.108.103','ns_1@172.23.120.245',

'ns_1@172.23.121.117','ns_1@172.23.121.3',

'ns_1@172.23.123.27','ns_1@172.23.123.28',

'ns_1@172.23.96.148','ns_1@172.23.96.251',

'ns_1@172.23.96.252','ns_1@172.23.96.253',

'ns_1@172.23.97.119','ns_1@172.23.97.121',

'ns_1@172.23.97.122','ns_1@172.23.97.239',

'ns_1@172.23.97.242','ns_1@172.23.98.135',

'ns_1@172.23.99.11','ns_1@172.23.99.20',

'ns_1@172.23.99.21','ns_1@172.23.99.25'], EjectNodes = ['ns_1@172.23.106.100'], Failed over and being ejected nodes = []; no delta recovery nodes; Operation Id = ad0efd005e57276e9944f0b3f58a8e0e hide

ns_server<0.121.0>: 2021-02-16 18:16:38.229749 crash_report        #{label=>{proc_lib,crash},report=>[[{initial_call,{ns_memcached,init,['Argument__1']}},{pid,<0.25327.1>},{registered_name,'ns_memcached-bucket8'},{error_info,{exit,{{badmatch,{error,closed}},[{mc_client_binary,cmd_vocal_recv,5,[{file,"src/mc_client_binary.erl"},{line,158}]},{mc_client_binary,set_collections_manifest,2,[{file,"src/mc_client_binary.erl"},{line,908}]},{memcached_bucket_config,ensure_collections,2,[{file,"src/memcached_bucket_config.erl"},{line,280}]},{ns_memcached,'-run_check_and_maybe_update_config/2-fun-0-',2,[{file,"src/ns_memcached.erl"},{line,874}]},{ns_memcached,'-perform_very_long_call/3-fun-0-',2,[{file,"src/ns_memcached.erl"},{line,324}]},{ns_memcached_sockets_pool,'-executing_on_socket/3-fun-0-',3,[{file,"src/ns_memcached_sockets_pool.erl"},{line,92}]},{async,'-async_init/4-fun-1-',3,[{file,"src/async.erl"},{line,197}]}]},[{gen_server,handle_common_reply,8,[{file,"gen_server.erl"},{line,751}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,249}]}]}},{ancestors,[<0.25326.1>,'single_bucket_kv_sup-bucket8',ns_bucket_sup,ns_bucket_worker_sup,ns_server_sup,ns_server_nodes_sup,<0.19728.0>,ns_server_cluster_sup,root_sup,<0.139.0>]},{message_queue_len,1},{messages,[check_config]},{links,[<0.25344.1>,<0.25346.1>,#Port<0.3827>,<0.25347.1>,<0.25345.1>,<0.25342.1>,<0.25343.1>,<0.25326.1>]},{dictionary,[]},{trap_exit,true},{status,running},{heap_size,514838},{stack_size,27},{reductions,266436738}],[{neighbour,[{pid,<0.25343.1>},{registered_name,[]},{initial_call,{erlang,apply,['Argument__1','Argument__2']}},{current_function,{gen,do_call,4}},{ancestors,['ns_memcached-bucket8',<0.25326.1>,'single_bucket_kv_sup-bucket8',ns_bucket_sup,ns_bucket_worker_sup,ns_server_sup,ns_server_nodes_sup,<0.19728.0>,ns_server_cluster_sup,root_sup,<0.139.0>]},{message_queue_len,0},{links,[<0.25327.1>,#Port<0.3829>]},{trap_exit,false},{status,waiting},{heap_size,987},{stack_size,22},{reductions,790570856},{current_stacktrace,[{gen,do_call,4,[{file,"gen.erl"},{line,167}]},{gen_server,call,3,[{file,"gen_server.erl"},{line,219}]},{ns_memcached,worker_loop,3,[{file,"src/ns_memcached.erl"},{line,204}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,249}]}]}]},{neighbour,[{pid,<0.25342.1>},{registered_name,[]},{initial_call,{erlang,apply,['Argument__1','Argument__2']}},{current_function,{gen,do_call,4}},{ancestors,['ns_memcached-bucket8',<0.25326.1>,'single_bucket_kv_sup-bucket8',ns_bucket_sup,ns_bucket_worker_sup,ns_server_sup,ns_server_nodes_sup,<0.19728.0>,ns_server_cluster_sup,root_sup,<0.139.0>]},{message_queue_len,0},{links,[<0.25327.1>,#Port<0.3828>]},{trap_exit,false},{status,waiting},{heap_size,6772},{stack_size,22},{reductions,2363259063},{current_stacktrace,[{gen,do_call,4,[{file,"gen.erl"},{line,167}]},{gen_server,call,3,[{file,"gen_server.erl"},{line,219}]},{ns_memcached,worker_loop,3,[{file,"src/ns_memcached.erl"},{line,204}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,249}]}]}]},{neighbour,[{pid,<0.25345.1>},{registered_name,[]},{initial_call,{erlang,apply,['Argument__1','Argument__2']}},{current_function,{gen,do_call,4}},{ancestors,['ns_memcached-bucket8',<0.25326.1>,'single_bucket_kv_sup-bucket8',ns_bucket_sup,ns_bucket_worker_sup,ns_server_sup,ns_server_nodes_sup,<0.19728.0>,ns_server_cluster_sup,root_sup,<0.139.0>]},{message_queue_len,0},{links,[<0.25327.1>,#Port<0.3831>]},{trap_exit,false},{status,waiting},{heap_size,2586},{stack_size,22},{reductions,3316821263},{current_stacktrace,[{gen,do_call,4,[{file,"gen.erl"},{line,167}]},{gen_server,call,3,[{file,"gen_server.erl"},{line,219}]},{ns_memcached,worker_loop,3,[{file,"src/ns_memcached.erl"},{line,204}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,249}]}]}]},{neighbour,[{pid,<0.25344.1>},{registered_name,[]},{initial_call,{erlang,apply,['Argument__1','Argument__2']}},{current_function,{gen,do_call,4}},{ancestors,['ns_memcached-bucket8',<0.25326.1>,'single_bucket_kv_sup-bucket8',ns_bucket_sup,ns_bucket_worker_sup,ns_server_sup,ns_server_nodes_sup,<0.19728.0>,ns_server_cluster_sup,root_sup,<0.139.0>]},{message_queue_len,0},{links,[<0.25327.1>,#Port<0.3830>]},{trap_exit,false},{status,waiting},{heap_size,196650},{stack_size,22},{reductions,7407},{current_stacktrace,[{gen,do_call,4,[{file,"gen.erl"},{line,167}]},{gen_server,call,3,[{file,"gen_server.erl"},{line,219}]},{ns_memcached,worker_loop,3,[{file,"src/ns_memcached.erl"},{line,204}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,249}]}]}]}]]}

ns_server<0.121.0>: 2021-02-16 18:16:38.231946 supervisor_report   #{label=>{supervisor,child_terminated},report=>[{supervisor,{<0.25326.1>,ns_memcached_sup}},{errorContext,child_terminated},{reason,{{badmatch,{error,closed}},[{mc_client_binary,cmd_vocal_recv,5,[{file,"src/mc_client_binary.erl"},{line,158}]},{mc_client_binary,set_collections_manifest,2,[{file,"src/mc_client_binary.erl"},{line,908}]},{memcached_bucket_config,ensure_collections,2,[{file,"src/memcached_bucket_config.erl"},{line,280}]},{ns_memcached,'-run_check_and_maybe_update_config/2-fun-0-',2,[{file,"src/ns_memcached.erl"},{line,874}]},{ns_memcached,'-perform_very_long_call/3-fun-0-',2,[{file,"src/ns_memcached.erl"},{line,324}]},{ns_memcached_sockets_pool,'-executing_on_socket/3-fun-0-',3,[{file,"src/ns_memcached_sockets_pool.erl"},{line,92}]},{async,'-async_init/4-fun-1-',3,[{file,"src/async.erl"},{line,197}]}]}},{offender,[{pid,<0.25327.1>},{id,{ns_memcached,"bucket8"}},{mfargs,{ns_memcached,start_link,["bucket8"]}},{restart_type,permanent},{shutdown,86400000},{child_type,worker}]}]}

[ns_server:info,2021-02-16T19:10:12.922-08:00,babysitter_of_ns_1@cb.local:<0.128.0>:ns_port_server:log:224]memcached<0.128.0>: W0216 19:10:12.719755 13779 HazptrDomain.h:671] Using the default inline executor for asynchronous reclamation may be susceptible to deadlock if the current thread happens to hold a resource needed by the deleter of a reclaimable object