[ns_server:debug,2014-04-21T1:42:39.703,ns_1@172.23.106.196:replication_manager-default<0.20330.7>:replication_manager:terminate:98]Replication manager died {{{case_clause, {error, {{{badmatch, {error, {{badmatch,{error,closed}}, [{mc_client_binary,cmd_vocal_recv,5}, {mc_client_binary,cmd_vocal,3}, {upr_commands,open_connection,3}, {upr_proxy,connect,4}, {upr_proxy,maybe_connect,1}, {upr_consumer_conn,init,2}, {upr_proxy,init,1}, {gen_server,init_it,6}]}}}, [{upr_replicator,init,1}, {gen_server,init_it,6}, {proc_lib,init_p_do_apply,3}]}, {child,undefined,'ns_1@172.23.106.198', {upr_replicator,start_link, ['ns_1@172.23.106.198',"default"]}, temporary,60000,worker, [upr_replicator]}}}}, [{upr_sup,maybe_start_replicator,2}, {upr_sup,setup_replication,3}, {upr_sup, '-set_desired_replications/2-lc$^0/1-0-',2}, {upr_sup, '-set_desired_replications/2-lc$^0/1-0-',2}, {replication_manager,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]}, {state,"default",upr,undefined,[],undefined}} [error_logger:error,2014-04-21T1:42:39.704,ns_1@172.23.106.196:error_logger<0.6.0>:ale_error_logger_handler:log_report:115] =========================CRASH REPORT========================= crasher: initial call: upr_replicator:init/1 pid: <0.20770.7> registered_name: [] exception exit: {{badmatch,{error,{{badmatch,{error,closed}}, [{mc_client_binary,cmd_vocal_recv,5}, {mc_client_binary,cmd_vocal,3}, {upr_commands,open_connection,3}, {upr_proxy,connect,4}, {upr_proxy,maybe_connect,1}, {upr_consumer_conn,init,2}, {upr_proxy,init,1}, {gen_server,init_it,6}]}}}, [{upr_replicator,init,1}, {gen_server,init_it,6}, {proc_lib,init_p_do_apply,3}]} in function gen_server:init_it/6 ancestors: ['upr_sup-default','single_bucket_sup-default',<0.20315.7>] messages: [] links: [<0.20332.7>,<0.20772.7>] dictionary: [] trap_exit: true status: running heap_size: 610 stack_size: 24 reductions: 154 neighbours: [error_logger:error,2014-04-21T1:42:39.705,ns_1@172.23.106.196:error_logger<0.6.0>:ale_error_logger_handler:log_report:115] =========================CRASH REPORT========================= crasher: initial call: upr_proxy:init/1 pid: <0.20772.7> registered_name: [] exception exit: {{badmatch,{error,closed}}, [{mc_client_binary,cmd_vocal_recv,5}, {mc_client_binary,cmd_vocal,3}, {upr_commands,open_connection,3}, {upr_proxy,connect,4}, {upr_proxy,maybe_connect,1}, {upr_consumer_conn,init,2}, {upr_proxy,init,1}, {gen_server,init_it,6}]} in function gen_server:init_it/6 ancestors: ['upr_replicator-default-ns_1@172.23.106.198', 'upr_sup-default','single_bucket_sup-default',<0.20315.7>] messages: [] links: [<0.20770.7>,#Port<0.423080>] dictionary: [] trap_exit: true status: running heap_size: 75025 stack_size: 24 reductions: 6369 neighbours: [error_logger:error,2014-04-21T1:42:39.708,ns_1@172.23.106.196:error_logger<0.6.0>:ale_error_logger_handler:log_report:115] =========================CRASH REPORT========================= crasher: initial call: replication_manager:init/1 pid: <0.20330.7> registered_name: 'replication_manager-default' exception exit: {{case_clause, {error, {{{badmatch, {error, {{badmatch,{error,closed}}, [{mc_client_binary,cmd_vocal_recv,5}, {mc_client_binary,cmd_vocal,3}, {upr_commands,open_connection,3}, {upr_proxy,connect,4}, {upr_proxy,maybe_connect,1}, {upr_consumer_conn,init,2}, {upr_proxy,init,1}, {gen_server,init_it,6}]}}}, [{upr_replicator,init,1}, {gen_server,init_it,6}, {proc_lib,init_p_do_apply,3}]}, {child,undefined,'ns_1@172.23.106.198', {upr_replicator,start_link, ['ns_1@172.23.106.198',"default"]}, temporary,60000,worker, [upr_replicator]}}}}, [{upr_sup,maybe_start_replicator,2}, {upr_sup,setup_replication,3}, {upr_sup,'-set_desired_replications/2-lc$^0/1-0-',2}, {upr_sup,'-set_desired_replications/2-lc$^0/1-0-',2}, {replication_manager,handle_call,3}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} in function gen_server:terminate/6 ancestors: ['single_bucket_sup-default',<0.20315.7>] messages: [] links: [<0.20316.7>] dictionary: [] trap_exit: true status: running heap_size: 2584 stack_size: 24 reductions: 6684 neighbours: [error_logger:error,2014-04-21T1:42:43.723,ns_1@172.23.106.196:error_logger<0.6.0>:ale_error_logger_handler:log_report:115] =========================CRASH REPORT========================= crasher: initial call: couch_set_view_group:init/1 pid: <0.22597.7> registered_name: [] exception exit: {badmatch,{error,closed}} in function couch_set_view_group:init/1 ancestors: [<0.22596.7>] messages: [{'EXIT',<0.22603.7>,closed}] links: [<0.22596.7>] dictionary: [] trap_exit: true status: running heap_size: 2584 stack_size: 24 reductions: 4564 neighbours: [error_logger:error,2014-04-21T1:42:43.723,ns_1@172.23.106.196:error_logger<0.6.0>:ale_error_logger_handler:log_msg:119]Supervisor received unexpected message: {ack,<0.22592.7>, {error, {bad_return_value, {badmatch,{error,closed}}}}} [error_logger:error,2014-04-21T1:42:43.724,ns_1@172.23.106.196:error_logger<0.6.0>:ale_error_logger_handler:log_report:115] =========================CRASH REPORT========================= crasher: initial call: capi_set_view_manager:init/1 pid: <0.22592.7> registered_name: 'capi_set_view_manager-default' exception exit: {bad_return_value,{badmatch,{error,closed}}} in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.20315.7>] messages: [replicate_newnodes_docs] links: [<0.22594.7>,<0.22595.7>,<0.22601.7>,<0.20316.7>] dictionary: [{ddoc_replication_proxy,<0.22595.7>}] trap_exit: true status: running heap_size: 10946 stack_size: 24 reductions: 3099 neighbours: neighbour: [{pid,<0.22601.7>}, {registered_name,[]}, {initial_call,{erlang,apply,2}}, {current_function, {capi_set_view_manager,nodeup_monitoring_loop,1}}, {ancestors,[]}, {messages,[]}, {links,[<0.22592.7>]}, {dictionary,[]}, {trap_exit,false}, {status,waiting}, {heap_size,233}, {stack_size,2}, {reductions,5}] neighbour: [{pid,<0.22595.7>}, {registered_name,'capi_ddoc_replication_srv-default'}, {initial_call,{erlang,apply,['Argument__1','Argument__2']}}, {current_function,{capi_ddoc_replication_srv,proxy_loop,1}}, {ancestors,['capi_set_view_manager-default', 'single_bucket_sup-default',<0.20315.7>]}, {messages,[]}, {links,[<0.22592.7>]}, {dictionary,[]}, {trap_exit,false}, {status,waiting}, {heap_size,233}, {stack_size,4}, {reductions,23}] =========================CRASH REPORT========================= crasher: initial call: couch_upr_client:init/1 pid: <0.22614.7> registered_name: [] exception exit: closed in function gen_server:init_it/6 ancestors: [<0.22609.7>,<0.22608.7>] messages: [] links: [<0.22609.7>,#Port<0.423879>] dictionary: [] trap_exit: false status: running heap_size: 233 stack_size: 24 reductions: 350 neighbours: [error_logger:error,2014-04-21T1:42:43.726,ns_1@172.23.106.196:error_logger<0.6.0>:ale_error_logger_handler:log_report:115] =========================CRASH REPORT========================= crasher: initial call: couch_set_view_group:init/1 pid: <0.22609.7> registered_name: [] exception exit: {badmatch,{error,closed}} in function couch_set_view_group:init/1 ancestors: [<0.22608.7>] messages: [{'EXIT',<0.22614.7>,closed}] links: [<0.22608.7>] dictionary: [] trap_exit: true status: running heap_size: 2584 stack_size: 24 reductions: 4564 neighbours: [views:debug,2014-04-21T1:42:43.726,ns_1@172.23.106.196:capi_set_view_manager-default<0.22626.7>:capi_set_view_manager:maybe_define_group:361] Calling couch_set_view:define_group(mapreduce_view, <<"default">>, <<"_design/test_view-504e77c">>, [{set_view_params,1024,[],[],true}]) [error_logger:error,2014-04-21T1:42:43.727,ns_1@172.23.106.196:error_logger<0.6.0>:ale_error_logger_handler:log_report:115] =========================CRASH REPORT========================= crasher: initial call: capi_set_view_manager:init/1 pid: <0.22604.7> registered_name: 'capi_set_view_manager-default' exception exit: {bad_return_value,{badmatch,{error,closed}}} in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.20315.7>] messages: [replicate_newnodes_docs] links: [<0.22605.7>,<0.22606.7>,<0.22607.7>,<0.20316.7>] dictionary: [{ddoc_replication_proxy,<0.22606.7>}] trap_exit: true status: running heap_size: 10946 stack_size: 24 reductions: 3099 neighbours: neighbour: [{pid,<0.22607.7>}, {registered_name,[]}, {initial_call,{erlang,apply,2}}, {current_function, {capi_set_view_manager,nodeup_monitoring_loop,1}}, {ancestors,[]}, {messages,[]}, {links,[<0.22604.7>]}, {dictionary,[]}, {trap_exit,false}, {status,waiting}, {heap_size,233}, {stack_size,2}, {reductions,5}] neighbour: [{pid,<0.22606.7>}, {registered_name,'capi_ddoc_replication_srv-default'}, {initial_call,{erlang,apply,['Argument__1','Argument__2']}}, {current_function,{capi_ddoc_replication_srv,proxy_loop,1}}, {ancestors,['capi_set_view_manager-default', 'single_bucket_sup-default',<0.20315.7>]}, {messages,[]}, {links,[<0.22604.7>]}, {dictionary,[]}, {trap_exit,false}, {status,waiting}, {heap_size,233}, {stack_size,4}, {reductions,23}] [error_logger:error,2014-04-21T1:42:43.727,ns_1@172.23.106.196:error_logger<0.6.0>:ale_error_logger_handler:log_msg:119]Supervisor received unexpected message: {ack,<0.22604.7>, {error, {bad_return_value, {badmatch,{error,closed}}}}} =========================CRASH REPORT========================= crasher: initial call: capi_set_view_manager:init/1 pid: <0.22714.7> registered_name: 'capi_set_view_manager-default' exception exit: {bad_return_value,{badmatch,{error,closed}}} in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.20315.7>] messages: [replicate_newnodes_docs, {'$gen_cast', {replicated_update, {doc,<<"_design/test_view-504e77c">>, {1,<<153,200,18,208>>}, <<"{\"views\":{\"test_view-504e77c\":{\"map\":\"function (doc) { if(doc.job_title !== undefined) emit([doc.join_yr, doc.join_mo, doc.join_day], [doc.name, doc.email] ); }\"}}}">>, 0,false,[]}}}, {'$gen_cast', {replicated_update, {doc,<<"_design/test_view-504e77c">>, {1,<<153,200,18,208>>}, <<"{\"views\":{\"test_view-504e77c\":{\"map\":\"function (doc) { if(doc.job_title !== undefined) emit([doc.join_yr, doc.join_mo, doc.join_day], [doc.name, doc.email] ); }\"}}}">>, 0,false,[]}}}] links: [<0.22715.7>,<0.22716.7>,<0.22717.7>,<0.20316.7>] dictionary: [{ddoc_replication_proxy,<0.22716.7>}] trap_exit: true status: running heap_size: 10946 stack_size: 24 reductions: 3114 neighbours: neighbour: [{pid,<0.22717.7>}, {registered_name,[]}, {initial_call,{erlang,apply,2}}, {current_function, {capi_set_view_manager,nodeup_monitoring_loop,1}}, {ancestors,[]}, {messages,[]}, {links,[<0.22714.7>]}, {dictionary,[]}, {trap_exit,false}, {status,waiting}, {heap_size,233}, {stack_size,2}, {reductions,5}] neighbour: [{pid,<0.22716.7>}, {registered_name,'capi_ddoc_replication_srv-default'}, {initial_call,{erlang,apply,['Argument__1','Argument__2']}}, {current_function,{capi_ddoc_replication_srv,proxy_loop,1}}, {ancestors,['capi_set_view_manager-default', 'single_bucket_sup-default',<0.20315.7>]}, {messages,[]}, {links,[<0.22714.7>]}, {dictionary,[]}, {trap_exit,false}, {status,waiting}, {heap_size,233}, {stack_size,4}, {reductions,27}] [error_logger:error,2014-04-21T1:42:50.416,ns_1@172.23.106.196:error_logger<0.6.0>:ale_error_logger_handler:log_report:115] =========================CRASH REPORT========================= crasher: initial call: erlang:apply/2 pid: <0.23030.7> registered_name: [] exception error: no match of right hand side value {error, couldnt_connect_to_memcached} in function ns_memcached:do_worker_init/1 in call from ns_memcached:worker_init/2 ancestors: ['ns_memcached-default','single_bucket_sup-default', <0.22992.7>] messages: [] links: [#Port<0.424063>,<0.22998.7>,#Port<0.424070>,#Port<0.424059>, #Port<0.424062>,#Port<0.424046>] dictionary: [] trap_exit: false status: running heap_size: 196418 stack_size: 24 reductions: 17656 neighbours: [user:info,2014-04-21T1:42:50.417,ns_1@172.23.106.196:ns_memcached-default<0.22998.7>:ns_memcached:terminate:821]Control connection to memcached on 'ns_1@172.23.106.196' disconnected: {badmatch, {error, couldnt_connect_to_memcached}} [error_logger:error,2014-04-21T1:42:50.419,ns_1@172.23.106.196:error_logger<0.6.0>:ale_error_logger_handler:log_msg:119]** Generic server <0.22998.7> terminating ** Last message in was {'EXIT',<0.23030.7>, {badmatch,{error,couldnt_connect_to_memcached}}} ** When Server state == {state,1,0,0, {[],[]}, {[],[]}, {[],[]}, connected, {1398,69765,399403}, "default",#Port<0.424036>, {interval,#Ref<0.0.28.134215>}, [{<0.23031.7>,#Ref<0.0.28.136461>}, {<0.23029.7>,#Ref<0.0.28.134508>}, {<0.23032.7>,#Ref<0.0.28.134245>}], []} ** Reason for termination == ** {badmatch,{error,couldnt_connect_to_memcached}}[error_logger:error,2014-04-21T1:42:50.420,ns_1@172.23.106.196:error_logger<0.6.0>:ale_error_logger_handler:log_report:115] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.22998.7> registered_name: [] exception exit: {badmatch,{error,couldnt_connect_to_memcached}} in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.22992.7>] messages: [] links: [<0.23029.7>,<0.23031.7>,<0.23032.7>,<0.307.0>,<0.22993.7>] dictionary: [] trap_exit: true status: running heap_size: 196418 stack_size: 24 reductions: 22902 neighbours: neighbour: [{pid,<0.23032.7>}, {registered_name,[]}, {initial_call,{erlang,apply,['Argument__1','Argument__2']}}, {current_function,{gen,do_call,4}}, {ancestors,['ns_memcached-default', 'single_bucket_sup-default',<0.22992.7>]}, {messages,[]}, {links,[<0.22998.7>,#Port<0.424043>]}, {dictionary,[]}, {trap_exit,false}, {status,waiting}, {heap_size,46368}, {stack_size,24}, {reductions,4663}] neighbour: [{pid,<0.23031.7>}, {registered_name,[]}, {initial_call,{erlang,apply,['Argument__1','Argument__2']}}, {current_function,{gen,do_call,4}}, {ancestors,['ns_memcached-default', 'single_bucket_sup-default',<0.22992.7>]}, {messages,[]}, {links,[<0.22998.7>,#Port<0.424044>]}, {dictionary,[]}, {trap_exit,false}, {status,waiting}, {heap_size,10946}, {stack_size,24}, {reductions,44938}] neighbour: [{pid,<0.23029.7>}, {registered_name,[]}, {initial_call,{erlang,apply,['Argument__1','Argument__2']}}, {current_function,{gen,do_call,4}}, {ancestors,['ns_memcached-default', 'single_bucket_sup-default',<0.22992.7>]}, {messages,[]}, {links,[<0.22998.7>,#Port<0.424045>]}, {dictionary,[]}, {trap_exit,false}, {status,waiting}, {heap_size,10946}, {stack_size,24}, {reductions,11546}]