[ns_server:error,2018-09-10T12:44:41.695-07:00,ns_1@172.23.100.204:<0.3398.0>:menelaus_web:loop:143]Server error during processing: ["web request failed", {path,"/saslauthd-saslauthd-port"}, {method,"RPCCONNECT"}, {type,error}, {what,function_clause}, {trace, [{misc,compare_secure, [undefined,"undefined"], [{file,"src/misc.erl"},{line,2498}]}, {ns_config_auth,authenticate,3, [{file,"src/ns_config_auth.erl"}, {line,110}]}, {ns_config_auth,authenticate,2, [{file,"src/ns_config_auth.erl"}, {line,116}]}, {menelaus_auth,authenticate,1, [{file,"src/menelaus_auth.erl"}, {line,295}]}, {menelaus_auth,verify_rest_auth,2, [{file,"src/menelaus_auth.erl"}, {line,387}]}, {menelaus_web,perform_action,2, [{file,"src/menelaus_web.erl"}, {line,899}]}, {request_throttler,do_request,3, [{file,"src/request_throttler.erl"}, {line,59}]}, {menelaus_web,loop,2, [{file,"src/menelaus_web.erl"}, {line,121}]}]}] [ns_server:error,2018-09-10T12:44:41.780-07:00,ns_1@172.23.100.204:<0.3410.0>:menelaus_web:loop:143]Server error during processing: ["web request failed", {path,"/goxdcr-cbauth"}, {method,"RPCCONNECT"}, {type,error}, {what,function_clause}, {trace, [{misc,compare_secure, [undefined,"undefined"], [{file,"src/misc.erl"},{line,2498}]}, {ns_config_auth,authenticate,3, [{file,"src/ns_config_auth.erl"}, {line,110}]}, {ns_config_auth,authenticate,2, [{file,"src/ns_config_auth.erl"}, {line,116}]}, {menelaus_auth,authenticate,1, [{file,"src/menelaus_auth.erl"}, {line,295}]}, {menelaus_auth,verify_rest_auth,2, [{file,"src/menelaus_auth.erl"}, {line,387}]}, {menelaus_web,perform_action,2, [{file,"src/menelaus_web.erl"}, {line,899}]}, {request_throttler,do_request,3, [{file,"src/request_throttler.erl"}, {line,59}]}, {menelaus_web,loop,2, [{file,"src/menelaus_web.erl"}, {line,121}]}]}] [ns_server:error,2018-09-10T12:44:42.698-07:00,ns_1@172.23.100.204:<0.3421.0>:menelaus_web:loop:143]Server error during processing: ["web request failed", {path,"/saslauthd-saslauthd-port"}, {method,"RPCCONNECT"}, {type,error}, {what,function_clause}, {trace, [{misc,compare_secure, [undefined,"undefined"], [{file,"src/misc.erl"},{line,2498}]}, {ns_config_auth,authenticate,3, [{file,"src/ns_config_auth.erl"}, {line,110}]}, {ns_config_auth,authenticate,2, [{file,"src/ns_config_auth.erl"}, {line,116}]}, {menelaus_auth,authenticate,1, [{file,"src/menelaus_auth.erl"}, {line,295}]}, {menelaus_auth,verify_rest_auth,2, [{file,"src/menelaus_auth.erl"}, {line,387}]}, {menelaus_web,perform_action,2, [{file,"src/menelaus_web.erl"}, {line,899}]}, {request_throttler,do_request,3, [{file,"src/request_throttler.erl"}, {line,59}]}, {menelaus_web,loop,2, [{file,"src/menelaus_web.erl"}, {line,121}]}]}] [ns_server:error,2018-09-10T12:44:42.783-07:00,ns_1@172.23.100.204:<0.3423.0>:menelaus_web:loop:143]Server error during processing: ["web request failed", {path,"/goxdcr-cbauth"}, {method,"RPCCONNECT"}, {type,error}, {what,function_clause}, {trace, [{misc,compare_secure, [undefined,"undefined"], [{file,"src/misc.erl"},{line,2498}]}, {ns_config_auth,authenticate,3, [{file,"src/ns_config_auth.erl"}, {line,110}]}, {ns_config_auth,authenticate,2, [{file,"src/ns_config_auth.erl"}, {line,116}]}, {menelaus_auth,authenticate,1, [{file,"src/menelaus_auth.erl"}, {line,295}]}, {menelaus_auth,verify_rest_auth,2, [{file,"src/menelaus_auth.erl"}, {line,387}]}, {menelaus_web,perform_action,2, [{file,"src/menelaus_web.erl"}, {line,899}]}, {request_throttler,do_request,3, [{file,"src/request_throttler.erl"}, {line,59}]}, {menelaus_web,loop,2, [{file,"src/menelaus_web.erl"}, {line,121}]}]}] [ns_server:error,2018-09-10T12:44:43.701-07:00,ns_1@172.23.100.204:<0.3499.0>:menelaus_web:loop:143]Server error during processing: ["web request failed", {path,"/saslauthd-saslauthd-port"}, {method,"RPCCONNECT"}, {type,error}, {what,function_clause}, {trace, [{misc,compare_secure, [undefined,"undefined"], [{file,"src/misc.erl"},{line,2498}]}, {ns_config_auth,authenticate,3, [{file,"src/ns_config_auth.erl"}, {line,110}]}, {ns_config_auth,authenticate,2, [{file,"src/ns_config_auth.erl"}, {line,116}]}, {menelaus_auth,authenticate,1, [{file,"src/menelaus_auth.erl"}, {line,295}]}, {menelaus_auth,verify_rest_auth,2, [{file,"src/menelaus_auth.erl"}, {line,387}]}, {menelaus_web,perform_action,2, [{file,"src/menelaus_web.erl"}, {line,899}]}, {request_throttler,do_request,3, [{file,"src/request_throttler.erl"}, {line,59}]}, {menelaus_web,loop,2, [{file,"src/menelaus_web.erl"}, {line,121}]}]}] [ns_server:error,2018-09-10T12:44:43.786-07:00,ns_1@172.23.100.204:<0.3549.0>:menelaus_web:loop:143]Server error during processing: ["web request failed", {path,"/goxdcr-cbauth"}, {method,"RPCCONNECT"}, {type,error}, {what,function_clause}, {trace, [{misc,compare_secure, [undefined,"undefined"], [{file,"src/misc.erl"},{line,2498}]}, {ns_config_auth,authenticate,3, [{file,"src/ns_config_auth.erl"}, {line,110}]}, {ns_config_auth,authenticate,2, [{file,"src/ns_config_auth.erl"}, {line,116}]}, {menelaus_auth,authenticate,1, [{file,"src/menelaus_auth.erl"}, {line,295}]}, {menelaus_auth,verify_rest_auth,2, [{file,"src/menelaus_auth.erl"}, {line,387}]}, {menelaus_web,perform_action,2, [{file,"src/menelaus_web.erl"}, {line,899}]}, {request_throttler,do_request,3, [{file,"src/request_throttler.erl"}, {line,59}]}, {menelaus_web,loop,2, [{file,"src/menelaus_web.erl"}, {line,121}]}]}] [ns_server:error,2018-09-10T12:44:44.703-07:00,ns_1@172.23.100.204:<0.3552.0>:menelaus_web:loop:143]Server error during processing: ["web request failed", {path,"/saslauthd-saslauthd-port"}, {method,"RPCCONNECT"}, {type,error}, {what,function_clause}, {trace, [{misc,compare_secure, [undefined,"undefined"], [{file,"src/misc.erl"},{line,2498}]}, {ns_config_auth,authenticate,3, [{file,"src/ns_config_auth.erl"}, {line,110}]}, {ns_config_auth,authenticate,2, [{file,"src/ns_config_auth.erl"}, {line,116}]}, {menelaus_auth,authenticate,1, [{file,"src/menelaus_auth.erl"}, {line,295}]}, {menelaus_auth,verify_rest_auth,2, [{file,"src/menelaus_auth.erl"}, {line,387}]}, {menelaus_web,perform_action,2, [{file,"src/menelaus_web.erl"}, {line,899}]}, {request_throttler,do_request,3, [{file,"src/request_throttler.erl"}, {line,59}]}, {menelaus_web,loop,2, [{file,"src/menelaus_web.erl"}, {line,121}]}]}] [ns_server:error,2018-09-10T12:44:44.789-07:00,ns_1@172.23.100.204:<0.3434.0>:menelaus_web:loop:143]Server error during processing: ["web request failed", {path,"/goxdcr-cbauth"}, {method,"RPCCONNECT"}, {type,error}, {what,function_clause}, {trace, [{misc,compare_secure, [undefined,"undefined"], [{file,"src/misc.erl"},{line,2498}]}, {ns_config_auth,authenticate,3, [{file,"src/ns_config_auth.erl"}, {line,110}]}, {ns_config_auth,authenticate,2, [{file,"src/ns_config_auth.erl"}, {line,116}]}, {menelaus_auth,authenticate,1, [{file,"src/menelaus_auth.erl"}, {line,295}]}, {menelaus_auth,verify_rest_auth,2, [{file,"src/menelaus_auth.erl"}, {line,387}]}, {menelaus_web,perform_action,2, [{file,"src/menelaus_web.erl"}, {line,899}]}, {request_throttler,do_request,3, [{file,"src/request_throttler.erl"}, {line,59}]}, {menelaus_web,loop,2, [{file,"src/menelaus_web.erl"}, {line,121}]}]}] [stats:error,2018-09-10T12:44:45.539-07:00,ns_1@172.23.100.204:<0.2838.0>:base_stats_collector:handle_info:109](Collector: global_stats_collector) Exception in stats collector: {error, {badmatch, {error, couldnt_connect_to_memcached}}, [{global_stats_collector, grab_stats, 1, [{file, "src/global_stats_collector.erl"}, {line, 35}]}, {base_stats_collector, handle_info, 2, [{file, "src/base_stats_collector.erl"}, {line, 89}]}, {gen_server, handle_msg, 5, [{file, "gen_server.erl"}, {line, 604}]}, {proc_lib, init_p_do_apply, 3, [{file, "proc_lib.erl"}, {line, 239}]}]} [ns_server:error,2018-09-10T12:44:45.706-07:00,ns_1@172.23.100.204:<0.3598.0>:menelaus_web:loop:143]Server error during processing: ["web request failed", {path,"/saslauthd-saslauthd-port"}, {method,"RPCCONNECT"}, {type,error}, {what,function_clause}, {trace, [{misc,compare_secure, [undefined,"undefined"], [{file,"src/misc.erl"},{line,2498}]}, {ns_config_auth,authenticate,3, [{file,"src/ns_config_auth.erl"}, {line,110}]}, {ns_config_auth,authenticate,2, [{file,"src/ns_config_auth.erl"}, {line,116}]}, {menelaus_auth,authenticate,1, [{file,"src/menelaus_auth.erl"}, {line,295}]}, {menelaus_auth,verify_rest_auth,2, [{file,"src/menelaus_auth.erl"}, {line,387}]}, {menelaus_web,perform_action,2, [{file,"src/menelaus_web.erl"}, {line,899}]}, {request_throttler,do_request,3, [{file,"src/request_throttler.erl"}, {line,59}]}, {menelaus_web,loop,2, [{file,"src/menelaus_web.erl"}, {line,121}]}]}] [ns_server:error,2018-09-10T12:44:45.792-07:00,ns_1@172.23.100.204:<0.3646.0>:menelaus_web:loop:143]Server error during processing: ["web request failed", {path,"/goxdcr-cbauth"}, {method,"RPCCONNECT"}, {type,error}, {what,function_clause}, {trace, [{misc,compare_secure, [undefined,"undefined"], [{file,"src/misc.erl"},{line,2498}]}, {ns_config_auth,authenticate,3, [{file,"src/ns_config_auth.erl"}, {line,110}]}, {ns_config_auth,authenticate,2, [{file,"src/ns_config_auth.erl"}, {line,116}]}, {menelaus_auth,authenticate,1, [{file,"src/menelaus_auth.erl"}, {line,295}]}, {menelaus_auth,verify_rest_auth,2, [{file,"src/menelaus_auth.erl"}, {line,387}]}, {menelaus_web,perform_action,2, [{file,"src/menelaus_web.erl"}, {line,899}]}, {request_throttler,do_request,3, [{file,"src/request_throttler.erl"}, {line,59}]}, {menelaus_web,loop,2, [{file,"src/menelaus_web.erl"}, {line,121}]}]}] [ns_server:error,2018-09-10T12:44:46.709-07:00,ns_1@172.23.100.204:<0.3647.0>:menelaus_web:loop:143]Server error during processing: ["web request failed", {path,"/saslauthd-saslauthd-port"}, {method,"RPCCONNECT"}, {type,error}, {what,function_clause}, {trace, [{misc,compare_secure, [undefined,"undefined"], [{file,"src/misc.erl"},{line,2498}]}, {ns_config_auth,authenticate,3, [{file,"src/ns_config_auth.erl"}, {line,110}]}, {ns_config_auth,authenticate,2, [{file,"src/ns_config_auth.erl"}, {line,116}]}, {menelaus_auth,authenticate,1, [{file,"src/menelaus_auth.erl"}, {line,295}]}, {menelaus_auth,verify_rest_auth,2, [{file,"src/menelaus_auth.erl"}, {line,387}]}, {menelaus_web,perform_action,2, [{file,"src/menelaus_web.erl"}, {line,899}]}, {request_throttler,do_request,3, [{file,"src/request_throttler.erl"}, {line,59}]}, {menelaus_web,loop,2, [{file,"src/menelaus_web.erl"}, {line,121}]}]}] [ns_server:error,2018-09-10T12:44:46.795-07:00,ns_1@172.23.100.204:<0.2764.0>:menelaus_web:loop:143]Server error during processing: ["web request failed", {path,"/goxdcr-cbauth"}, {method,"RPCCONNECT"}, {type,error}, {what,function_clause}, {trace, [{misc,compare_secure, [undefined,"undefined"], [{file,"src/misc.erl"},{line,2498}]}, {ns_config_auth,authenticate,3, [{file,"src/ns_config_auth.erl"}, {line,110}]}, {ns_config_auth,authenticate,2, [{file,"src/ns_config_auth.erl"}, {line,116}]}, {menelaus_auth,authenticate,1, [{file,"src/menelaus_auth.erl"}, {line,295}]}, {menelaus_auth,verify_rest_auth,2, [{file,"src/menelaus_auth.erl"}, {line,387}]}, {menelaus_web,perform_action,2, [{file,"src/menelaus_web.erl"}, {line,899}]}, {request_throttler,do_request,3, [{file,"src/request_throttler.erl"}, {line,59}]}, {menelaus_web,loop,2, [{file,"src/menelaus_web.erl"}, {line,121}]}]}] [ns_server:error,2018-09-10T12:44:47.712-07:00,ns_1@172.23.100.204:<0.3681.0>:menelaus_web:loop:143]Server error during processing: ["web request failed", {path,"/saslauthd-saslauthd-port"}, {method,"RPCCONNECT"}, {type,error}, {what,function_clause}, {trace, [{misc,compare_secure, [undefined,"undefined"], [{file,"src/misc.erl"},{line,2498}]}, {ns_config_auth,authenticate,3, [{file,"src/ns_config_auth.erl"}, {line,110}]}, {ns_config_auth,authenticate,2, [{file,"src/ns_config_auth.erl"}, {line,116}]}, {menelaus_auth,authenticate,1, [{file,"src/menelaus_auth.erl"}, {line,295}]}, {menelaus_auth,verify_rest_auth,2, [{file,"src/menelaus_auth.erl"}, {line,387}]}, {menelaus_web,perform_action,2, [{file,"src/menelaus_web.erl"}, {line,899}]}, {request_throttler,do_request,3, [{file,"src/request_throttler.erl"}, {line,59}]}, {menelaus_web,loop,2, [{file,"src/menelaus_web.erl"}, {line,121}]}]}] [ns_server:error,2018-09-10T12:44:47.798-07:00,ns_1@172.23.100.204:<0.3708.0>:menelaus_web:loop:143]Server error during processing: ["web request failed", {path,"/goxdcr-cbauth"}, {method,"RPCCONNECT"}, {type,error}, {what,function_clause}, {trace, [{misc,compare_secure, [undefined,"undefined"], [{file,"src/misc.erl"},{line,2498}]}, {ns_config_auth,authenticate,3, [{file,"src/ns_config_auth.erl"}, {line,110}]}, {ns_config_auth,authenticate,2, [{file,"src/ns_config_auth.erl"}, {line,116}]}, {menelaus_auth,authenticate,1, [{file,"src/menelaus_auth.erl"}, {line,295}]}, {menelaus_auth,verify_rest_auth,2, [{file,"src/menelaus_auth.erl"}, {line,387}]}, {menelaus_web,perform_action,2, [{file,"src/menelaus_web.erl"}, {line,899}]}, {request_throttler,do_request,3, [{file,"src/request_throttler.erl"}, {line,59}]}, {menelaus_web,loop,2, [{file,"src/menelaus_web.erl"}, {line,121}]}]}] [ns_server:error,2018-09-10T12:44:48.714-07:00,ns_1@172.23.100.204:<0.3709.0>:menelaus_web:loop:143]Server error during processing: ["web request failed", {path,"/saslauthd-saslauthd-port"}, {method,"RPCCONNECT"}, {type,error}, {what,function_clause}, {trace, [{misc,compare_secure, [undefined,"undefined"], [{file,"src/misc.erl"},{line,2498}]}, {ns_config_auth,authenticate,3, [{file,"src/ns_config_auth.erl"}, {line,110}]}, {ns_config_auth,authenticate,2, [{file,"src/ns_config_auth.erl"}, {line,116}]}, {menelaus_auth,authenticate,1, [{file,"src/menelaus_auth.erl"}, {line,295}]}, {menelaus_auth,verify_rest_auth,2, [{file,"src/menelaus_auth.erl"}, {line,387}]}, {menelaus_web,perform_action,2, [{file,"src/menelaus_web.erl"}, {line,899}]}, {request_throttler,do_request,3, [{file,"src/request_throttler.erl"}, {line,59}]}, {menelaus_web,loop,2, [{file,"src/menelaus_web.erl"}, {line,121}]}]}] [ns_server:error,2018-09-10T12:44:48.801-07:00,ns_1@172.23.100.204:<0.2774.0>:menelaus_web:loop:143]Server error during processing: ["web request failed", {path,"/goxdcr-cbauth"}, {method,"RPCCONNECT"}, {type,error}, {what,function_clause}, {trace, [{misc,compare_secure, [undefined,"undefined"], [{file,"src/misc.erl"},{line,2498}]}, {ns_config_auth,authenticate,3, [{file,"src/ns_config_auth.erl"}, {line,110}]}, {ns_config_auth,authenticate,2, [{file,"src/ns_config_auth.erl"}, {line,116}]}, {menelaus_auth,authenticate,1, [{file,"src/menelaus_auth.erl"}, {line,295}]}, {menelaus_auth,verify_rest_auth,2, [{file,"src/menelaus_auth.erl"}, {line,387}]}, {menelaus_web,perform_action,2, [{file,"src/menelaus_web.erl"}, {line,899}]}, {request_throttler,do_request,3, [{file,"src/request_throttler.erl"}, {line,59}]}, {menelaus_web,loop,2, [{file,"src/menelaus_web.erl"}, {line,121}]}]}] [stats:error,2018-09-10T12:44:49.568-07:00,ns_1@172.23.100.204:<0.2838.0>:base_stats_collector:handle_info:109](Collector: global_stats_collector) Exception in stats collector: {error, {badmatch, {error, couldnt_connect_to_memcached}}, [{global_stats_collector, grab_stats, 1, [{file, "src/global_stats_collector.erl"}, {line, 35}]}, {base_stats_collector, handle_info, 2, [{file, "src/base_stats_collector.erl"}, {line, 89}]}, {gen_server, handle_msg, 5, [{file, "gen_server.erl"}, {line, 604}]}, {proc_lib, init_p_do_apply, 3, [{file, "proc_lib.erl"}, {line, 239}]}]} [ns_server:error,2018-09-10T12:44:49.716-07:00,ns_1@172.23.100.204:<0.3743.0>:menelaus_web:loop:143]Server error during processing: ["web request failed", {path,"/saslauthd-saslauthd-port"}, {method,"RPCCONNECT"}, {type,error}, {what,function_clause}, {trace, [{misc,compare_secure, [undefined,"undefined"], [{file,"src/misc.erl"},{line,2498}]}, {ns_config_auth,authenticate,3, [{file,"src/ns_config_auth.erl"}, {line,110}]}, {ns_config_auth,authenticate,2, [{file,"src/ns_config_auth.erl"}, {line,116}]}, {menelaus_auth,authenticate,1, [{file,"src/menelaus_auth.erl"}, {line,295}]}, {menelaus_auth,verify_rest_auth,2, [{file,"src/menelaus_auth.erl"}, {line,387}]}, {menelaus_web,perform_action,2, [{file,"src/menelaus_web.erl"}, {line,899}]}, {request_throttler,do_request,3, [{file,"src/request_throttler.erl"}, {line,59}]}, {menelaus_web,loop,2, [{file,"src/menelaus_web.erl"}, {line,121}]}]}] [ns_server:error,2018-09-10T12:44:49.804-07:00,ns_1@172.23.100.204:<0.3778.0>:menelaus_web:loop:143]Server error during processing: ["web request failed", {path,"/goxdcr-cbauth"}, {method,"RPCCONNECT"}, {type,error}, {what,function_clause}, {trace, [{misc,compare_secure, [undefined,"undefined"], [{file,"src/misc.erl"},{line,2498}]}, {ns_config_auth,authenticate,3, [{file,"src/ns_config_auth.erl"}, {line,110}]}, {ns_config_auth,authenticate,2, [{file,"src/ns_config_auth.erl"}, {line,116}]}, {menelaus_auth,authenticate,1, [{file,"src/menelaus_auth.erl"}, {line,295}]}, {menelaus_auth,verify_rest_auth,2, [{file,"src/menelaus_auth.erl"}, {line,387}]}, {menelaus_web,perform_action,2, [{file,"src/menelaus_web.erl"}, {line,899}]}, {request_throttler,do_request,3, [{file,"src/request_throttler.erl"}, {line,59}]}, {menelaus_web,loop,2, [{file,"src/menelaus_web.erl"}, {line,121}]}]}] [ns_server:error,2018-09-10T12:44:50.719-07:00,ns_1@172.23.100.204:<0.3779.0>:menelaus_web:loop:143]Server error during processing: ["web request failed", {path,"/saslauthd-saslauthd-port"}, {method,"RPCCONNECT"}, {type,error}, {what,function_clause}, {trace, [{misc,compare_secure, [undefined,"undefined"], [{file,"src/misc.erl"},{line,2498}]}, {ns_config_auth,authenticate,3, [{file,"src/ns_config_auth.erl"}, {line,110}]}, {ns_config_auth,authenticate,2, [{file,"src/ns_config_auth.erl"}, {line,116}]}, {menelaus_auth,authenticate,1, [{file,"src/menelaus_auth.erl"}, {line,295}]}, {menelaus_auth,verify_rest_auth,2, [{file,"src/menelaus_auth.erl"}, {line,387}]}, {menelaus_web,perform_action,2, [{file,"src/menelaus_web.erl"}, {line,899}]}, {request_throttler,do_request,3, [{file,"src/request_throttler.erl"}, {line,59}]}, {menelaus_web,loop,2, [{file,"src/menelaus_web.erl"}, {line,121}]}]}] [ns_server:error,2018-09-10T12:44:50.807-07:00,ns_1@172.23.100.204:<0.2772.0>:menelaus_web:loop:143]Server error during processing: ["web request failed", {path,"/goxdcr-cbauth"}, {method,"RPCCONNECT"}, {type,error}, {what,function_clause}, {trace, [{misc,compare_secure, [undefined,"undefined"], [{file,"src/misc.erl"},{line,2498}]}, {ns_config_auth,authenticate,3, [{file,"src/ns_config_auth.erl"}, {line,110}]}, {ns_config_auth,authenticate,2, [{file,"src/ns_config_auth.erl"}, {line,116}]}, {menelaus_auth,authenticate,1, [{file,"src/menelaus_auth.erl"}, {line,295}]}, {menelaus_auth,verify_rest_auth,2, [{file,"src/menelaus_auth.erl"}, {line,387}]}, {menelaus_web,perform_action,2, [{file,"src/menelaus_web.erl"}, {line,899}]}, {request_throttler,do_request,3, [{file,"src/request_throttler.erl"}, {line,59}]}, {menelaus_web,loop,2, [{file,"src/menelaus_web.erl"}, {line,121}]}]}] [ns_server:error,2018-09-10T12:44:51.721-07:00,ns_1@172.23.100.204:<0.3832.0>:menelaus_web:loop:143]Server error during processing: ["web request failed", {path,"/saslauthd-saslauthd-port"}, {method,"RPCCONNECT"}, {type,error}, {what,function_clause}, {trace, [{misc,compare_secure, [undefined,"undefined"], [{file,"src/misc.erl"},{line,2498}]}, {ns_config_auth,authenticate,3, [{file,"src/ns_config_auth.erl"}, {line,110}]}, {ns_config_auth,authenticate,2, [{file,"src/ns_config_auth.erl"}, {line,116}]}, {menelaus_auth,authenticate,1, [{file,"src/menelaus_auth.erl"}, {line,295}]}, {menelaus_auth,verify_rest_auth,2, [{file,"src/menelaus_auth.erl"}, {line,387}]}, {menelaus_web,perform_action,2, [{file,"src/menelaus_web.erl"}, {line,899}]}, {request_throttler,do_request,3, [{file,"src/request_throttler.erl"}, {line,59}]}, {menelaus_web,loop,2, [{file,"src/menelaus_web.erl"}, {line,121}]}]}] [ns_server:error,2018-09-10T12:44:51.810-07:00,ns_1@172.23.100.204:<0.3857.0>:menelaus_web:loop:143]Server error during processing: ["web request failed", {path,"/goxdcr-cbauth"}, {method,"RPCCONNECT"}, {type,error}, {what,function_clause}, {trace, [{misc,compare_secure, [undefined,"undefined"], [{file,"src/misc.erl"},{line,2498}]}, {ns_config_auth,authenticate,3, [{file,"src/ns_config_auth.erl"}, {line,110}]}, {ns_config_auth,authenticate,2, [{file,"src/ns_config_auth.erl"}, {line,116}]}, {menelaus_auth,authenticate,1, [{file,"src/menelaus_auth.erl"}, {line,295}]}, {menelaus_auth,verify_rest_auth,2, [{file,"src/menelaus_auth.erl"}, {line,387}]}, {menelaus_web,perform_action,2, [{file,"src/menelaus_web.erl"}, {line,899}]}, {request_throttler,do_request,3, [{file,"src/request_throttler.erl"}, {line,59}]}, {menelaus_web,loop,2, [{file,"src/menelaus_web.erl"}, {line,121}]}]}] [ns_server:error,2018-09-10T12:44:52.724-07:00,ns_1@172.23.100.204:<0.3858.0>:menelaus_web:loop:143]Server error during processing: ["web request failed", {path,"/saslauthd-saslauthd-port"}, {method,"RPCCONNECT"}, {type,error}, {what,function_clause}, {trace, [{misc,compare_secure, [undefined,"undefined"], [{file,"src/misc.erl"},{line,2498}]}, {ns_config_auth,authenticate,3, [{file,"src/ns_config_auth.erl"}, {line,110}]}, {ns_config_auth,authenticate,2, [{file,"src/ns_config_auth.erl"}, {line,116}]}, {menelaus_auth,authenticate,1, [{file,"src/menelaus_auth.erl"}, {line,295}]}, {menelaus_auth,verify_rest_auth,2, [{file,"src/menelaus_auth.erl"}, {line,387}]}, {menelaus_web,perform_action,2, [{file,"src/menelaus_web.erl"}, {line,899}]}, {request_throttler,do_request,3, [{file,"src/request_throttler.erl"}, {line,59}]}, {menelaus_web,loop,2, [{file,"src/menelaus_web.erl"}, {line,121}]}]}] [ns_server:error,2018-09-10T12:44:52.813-07:00,ns_1@172.23.100.204:<0.2773.0>:menelaus_web:loop:143]Server error during processing: ["web request failed", {path,"/goxdcr-cbauth"}, {method,"RPCCONNECT"}, {type,error}, {what,function_clause}, {trace, [{misc,compare_secure, [undefined,"undefined"], [{file,"src/misc.erl"},{line,2498}]}, {ns_config_auth,authenticate,3, [{file,"src/ns_config_auth.erl"}, {line,110}]}, {ns_config_auth,authenticate,2, [{file,"src/ns_config_auth.erl"}, {line,116}]}, {menelaus_auth,authenticate,1, [{file,"src/menelaus_auth.erl"}, {line,295}]}, {menelaus_auth,verify_rest_auth,2, [{file,"src/menelaus_auth.erl"}, {line,387}]}, {menelaus_web,perform_action,2, [{file,"src/menelaus_web.erl"}, {line,899}]}, {request_throttler,do_request,3, [{file,"src/request_throttler.erl"}, {line,59}]}, {menelaus_web,loop,2, [{file,"src/menelaus_web.erl"}, {line,121}]}]}] [stats:error,2018-09-10T12:44:53.601-07:00,ns_1@172.23.100.204:<0.2838.0>:base_stats_collector:handle_info:109](Collector: global_stats_collector) Exception in stats collector: {error, {badmatch, {error, couldnt_connect_to_memcached}}, [{global_stats_collector, grab_stats, 1, [{file, "src/global_stats_collector.erl"}, {line, 35}]}, {base_stats_collector, handle_info, 2, [{file, "src/base_stats_collector.erl"}, {line, 89}]}, {gen_server, handle_msg, 5, [{file, "gen_server.erl"}, {line, 604}]}, {proc_lib, init_p_do_apply, 3, [{file, "proc_lib.erl"}, {line, 239}]}]} [ns_server:error,2018-09-10T12:44:53.727-07:00,ns_1@172.23.100.204:<0.3890.0>:menelaus_web:loop:143]Server error during processing: ["web request failed", {path,"/saslauthd-saslauthd-port"}, {method,"RPCCONNECT"}, {type,error}, {what,function_clause}, {trace, [{misc,compare_secure, [undefined,"undefined"], [{file,"src/misc.erl"},{line,2498}]}, {ns_config_auth,authenticate,3, [{file,"src/ns_config_auth.erl"}, {line,110}]}, {ns_config_auth,authenticate,2, [{file,"src/ns_config_auth.erl"}, {line,116}]}, {menelaus_auth,authenticate,1, [{file,"src/menelaus_auth.erl"}, {line,295}]}, {menelaus_auth,verify_rest_auth,2, [{file,"src/menelaus_auth.erl"}, {line,387}]}, {menelaus_web,perform_action,2, [{file,"src/menelaus_web.erl"}, {line,899}]}, {request_throttler,do_request,3, [{file,"src/request_throttler.erl"}, {line,59}]}, {menelaus_web,loop,2, [{file,"src/menelaus_web.erl"}, {line,121}]}]}] [ns_server:error,2018-09-10T12:44:53.816-07:00,ns_1@172.23.100.204:<0.3917.0>:menelaus_web:loop:143]Server error during processing: ["web request failed", {path,"/goxdcr-cbauth"}, {method,"RPCCONNECT"}, {type,error}, {what,function_clause}, {trace, [{misc,compare_secure, [undefined,"undefined"], [{file,"src/misc.erl"},{line,2498}]}, {ns_config_auth,authenticate,3, [{file,"src/ns_config_auth.erl"}, {line,110}]}, {ns_config_auth,authenticate,2, [{file,"src/ns_config_auth.erl"}, {line,116}]}, {menelaus_auth,authenticate,1, [{file,"src/menelaus_auth.erl"}, {line,295}]}, {menelaus_auth,verify_rest_auth,2, [{file,"src/menelaus_auth.erl"}, {line,387}]}, {menelaus_web,perform_action,2, [{file,"src/menelaus_web.erl"}, {line,899}]}, {request_throttler,do_request,3, [{file,"src/request_throttler.erl"}, {line,59}]}, {menelaus_web,loop,2, [{file,"src/menelaus_web.erl"}, {line,121}]}]}] [ns_server:error,2018-09-10T12:44:55.483-07:00,ns_1@172.23.100.204:query_stats_collector<0.2833.0>:rest_utils:get_json_local:63]Request to (n1ql) /admin/stats failed: {error, {econnrefused, [{lhttpc_client,send_request,1, [{file, "/home/couchbase/jenkins/workspace/couchbase-server-unix/couchdb/src/lhttpc/lhttpc_client.erl"}, {line,220}]}, {lhttpc_client,execute,9, [{file, "/home/couchbase/jenkins/workspace/couchbase-server-unix/couchdb/src/lhttpc/lhttpc_client.erl"}, {line,169}]}, {lhttpc_client,request,9, [{file, "/home/couchbase/jenkins/workspace/couchbase-server-unix/couchdb/src/lhttpc/lhttpc_client.erl"}, {line,92}]}]}} [ns_server:error,2018-09-10T12:45:03.506-07:00,ns_1@172.23.100.204:service_rebalancer-index<0.4393.0>:service_rebalancer:run_rebalance:74]Got exit message from parent: {'EXIT',<0.4392.0>,shutdown} [stats:error,2018-09-10T12:45:05.755-07:00,ns_1@172.23.100.204:<0.4666.0>:stats_reader:log_bad_responses:233]Some nodes didn't respond: ['ns_1@172.23.100.204','ns_1@172.23.100.205', 'ns_1@172.23.100.206','ns_1@172.23.100.207'] [ns_server:error,2018-09-10T12:45:06.491-07:00,ns_1@172.23.100.204:service_agent-index<0.4134.0>:service_agent:handle_info:239]Lost json rpc connection for service index, reason shutdown. Terminating. [ns_server:error,2018-09-10T12:45:06.491-07:00,ns_1@172.23.100.204:service_agent-index<0.4134.0>:service_agent:terminate:260]Terminating abnormally [ns_server:error,2018-09-10T12:51:07.142-07:00,ns_1@172.23.100.204:query_stats_collector<0.2833.0>:rest_utils:get_json_local:63]Request to (n1ql) /admin/stats failed: {error, {econnrefused, [{lhttpc_client,send_request,1, [{file, "/home/couchbase/jenkins/workspace/couchbase-server-unix/couchdb/src/lhttpc/lhttpc_client.erl"}, {line,220}]}, {lhttpc_client,execute,9, [{file, "/home/couchbase/jenkins/workspace/couchbase-server-unix/couchdb/src/lhttpc/lhttpc_client.erl"}, {line,169}]}, {lhttpc_client,request,9, [{file, "/home/couchbase/jenkins/workspace/couchbase-server-unix/couchdb/src/lhttpc/lhttpc_client.erl"}, {line,92}]}]}} [ns_server:error,2018-09-10T12:51:11.902-07:00,ns_1@172.23.100.204:query_stats_collector<0.2833.0>:rest_utils:get_json_local:63]Request to (n1ql) /admin/stats failed: {error, {econnrefused, [{lhttpc_client,send_request,1, [{file, "/home/couchbase/jenkins/workspace/couchbase-server-unix/couchdb/src/lhttpc/lhttpc_client.erl"}, {line,220}]}, {lhttpc_client,execute,9, [{file, "/home/couchbase/jenkins/workspace/couchbase-server-unix/couchdb/src/lhttpc/lhttpc_client.erl"}, {line,169}]}, {lhttpc_client,request,9, [{file, "/home/couchbase/jenkins/workspace/couchbase-server-unix/couchdb/src/lhttpc/lhttpc_client.erl"}, {line,92}]}]}} [ns_server:error,2018-09-10T12:51:12.484-07:00,ns_1@172.23.100.204:query_stats_collector<0.2833.0>:rest_utils:get_json_local:63]Request to (n1ql) /admin/stats failed: {error, {econnrefused, [{lhttpc_client,send_request,1, [{file, "/home/couchbase/jenkins/workspace/couchbase-server-unix/couchdb/src/lhttpc/lhttpc_client.erl"}, {line,220}]}, {lhttpc_client,execute,9, [{file, "/home/couchbase/jenkins/workspace/couchbase-server-unix/couchdb/src/lhttpc/lhttpc_client.erl"}, {line,169}]}, {lhttpc_client,request,9, [{file, "/home/couchbase/jenkins/workspace/couchbase-server-unix/couchdb/src/lhttpc/lhttpc_client.erl"}, {line,92}]}]}} [ns_server:error,2018-09-10T12:51:13.483-07:00,ns_1@172.23.100.204:query_stats_collector<0.2833.0>:rest_utils:get_json_local:63]Request to (n1ql) /admin/stats failed: {error, {econnrefused, [{lhttpc_client,send_request,1, [{file, "/home/couchbase/jenkins/workspace/couchbase-server-unix/couchdb/src/lhttpc/lhttpc_client.erl"}, {line,220}]}, {lhttpc_client,execute,9, [{file, "/home/couchbase/jenkins/workspace/couchbase-server-unix/couchdb/src/lhttpc/lhttpc_client.erl"}, {line,169}]}, {lhttpc_client,request,9, [{file, "/home/couchbase/jenkins/workspace/couchbase-server-unix/couchdb/src/lhttpc/lhttpc_client.erl"}, {line,92}]}]}} [ns_server:error,2018-09-10T12:51:14.483-07:00,ns_1@172.23.100.204:query_stats_collector<0.2833.0>:rest_utils:get_json_local:63]Request to (n1ql) /admin/stats failed: {error, {econnrefused, [{lhttpc_client,send_request,1, [{file, "/home/couchbase/jenkins/workspace/couchbase-server-unix/couchdb/src/lhttpc/lhttpc_client.erl"}, {line,220}]}, {lhttpc_client,execute,9, [{file, "/home/couchbase/jenkins/workspace/couchbase-server-unix/couchdb/src/lhttpc/lhttpc_client.erl"}, {line,169}]}, {lhttpc_client,request,9, [{file, "/home/couchbase/jenkins/workspace/couchbase-server-unix/couchdb/src/lhttpc/lhttpc_client.erl"}, {line,92}]}]}} [ns_server:error,2018-09-10T12:51:15.484-07:00,ns_1@172.23.100.204:query_stats_collector<0.2833.0>:rest_utils:get_json_local:63]Request to (n1ql) /admin/stats failed: {error, {econnrefused, [{lhttpc_client,send_request,1, [{file, "/home/couchbase/jenkins/workspace/couchbase-server-unix/couchdb/src/lhttpc/lhttpc_client.erl"}, {line,220}]}, {lhttpc_client,execute,9, [{file, "/home/couchbase/jenkins/workspace/couchbase-server-unix/couchdb/src/lhttpc/lhttpc_client.erl"}, {line,169}]}, {lhttpc_client,request,9, [{file, "/home/couchbase/jenkins/workspace/couchbase-server-unix/couchdb/src/lhttpc/lhttpc_client.erl"}, {line,92}]}]}} [ns_server:error,2018-09-10T12:51:16.484-07:00,ns_1@172.23.100.204:query_stats_collector<0.2833.0>:rest_utils:get_json_local:63]Request to (n1ql) /admin/stats failed: {error, {econnrefused, [{lhttpc_client,send_request,1, [{file, "/home/couchbase/jenkins/workspace/couchbase-server-unix/couchdb/src/lhttpc/lhttpc_client.erl"}, {line,220}]}, {lhttpc_client,execute,9, [{file, "/home/couchbase/jenkins/workspace/couchbase-server-unix/couchdb/src/lhttpc/lhttpc_client.erl"}, {line,169}]}, {lhttpc_client,request,9, [{file, "/home/couchbase/jenkins/workspace/couchbase-server-unix/couchdb/src/lhttpc/lhttpc_client.erl"}, {line,92}]}]}}