Details
-
Bug
-
Resolution: Fixed
-
Critical
-
7.1.0
-
Untriaged
-
-
1
-
Unknown
Description
Build : 7.1.0-1472
Test : -test tests/integration/neo/test_neo_couchstore.yml -scope tests/integration/neo/scope_couchstore.yml
Scale : 3
Iteration : 1st
Few cluster operations in the Couchstore longevity test failed due to Internal Server error.
1. Rebalance to add 172.23.106.100. Error log on 172.23.108.103 -
[ns_server:error,2021-10-12T09:09:55.042-07:00,ns_1@172.23.108.103:<0.459.318>:menelaus_util:reply_server_error:207]Server error during processing: ["web request failed",
|
{path,"/pools/default/tasks"},
|
{method,'GET'},
|
{type,error},
|
{what,{badmatch,{error,timeout}}},
|
{trace,
|
[{goxdcr_rest,send_with_timeout,5,
|
[{file,"src/goxdcr_rest.erl"},{line,54}]},
|
{goxdcr_rest,query_goxdcr,4,
|
[{file,"src/goxdcr_rest.erl"},{line,112}]},
|
{goxdcr_rest,get_from_goxdcr,3,
|
[{file,"src/goxdcr_rest.erl"},{line,135}]},
|
{ns_doctor,build_tasks_list,2,
|
[{file,"src/ns_doctor.erl"},{line,222}]},
|
{menelaus_web_misc,do_handle_tasks,3,
|
[{file,"src/menelaus_web_misc.erl"},
|
{line,63}]},
|
{request_throttler,do_request,3,
|
[{file,"src/request_throttler.erl"},
|
{line,58}]},
|
{menelaus_util,handle_request,2,
|
[{file,"src/menelaus_util.erl"},
|
{line,218}]},
|
{mochiweb_http,headers,6,
|
[{file,
|
"/home/couchbase/jenkins/workspace/couchbase-server-unix/couchdb/src/mochiweb/mochiweb_http.erl"},
|
{line,150}]}]}]
|
2. Rebalance to add 172.23.96.148. Error log on 172.23.108.103 -
[ns_server:error,2021-10-12T09:34:23.527-07:00,ns_1@172.23.108.103:<0.1505.324>:menelaus_util:reply_server_error:207]Server error during processing: ["web request failed",
|
{path,"/pools/default/tasks"},
|
{method,'GET'},
|
{type,error},
|
{what,{badmatch,{error,timeout}}},
|
{trace,
|
[{goxdcr_rest,send_with_timeout,5,
|
[{file,"src/goxdcr_rest.erl"},{line,54}]},
|
{goxdcr_rest,query_goxdcr,4,
|
[{file,"src/goxdcr_rest.erl"},{line,112}]},
|
{goxdcr_rest,get_from_goxdcr,3,
|
[{file,"src/goxdcr_rest.erl"},{line,135}]},
|
{ns_doctor,build_tasks_list,2,
|
[{file,"src/ns_doctor.erl"},{line,222}]},
|
{menelaus_web_misc,do_handle_tasks,3,
|
[{file,"src/menelaus_web_misc.erl"},
|
{line,63}]},
|
{request_throttler,do_request,3,
|
[{file,"src/request_throttler.erl"},
|
{line,58}]},
|
{menelaus_util,handle_request,2,
|
[{file,"src/menelaus_util.erl"},
|
{line,218}]},
|
{mochiweb_http,headers,6,
|
[{file,
|
"/home/couchbase/jenkins/workspace/couchbase-server-unix/couchdb/src/mochiweb/mochiweb_http.erl"},
|
{line,150}]}]}
|
3. Rebalance at 2021-10-12T10:03:37. Error log on 172.23.108.103 -
[ns_server:error,2021-10-12T10:03:31.448-07:00,ns_1@172.23.108.103:<0.28755.330>:menelaus_util:reply_server_error:207]Server error during processing: ["web request failed",
|
{path,"/pools/default/tasks"},
|
{method,'GET'},
|
{type,error},
|
{what,{badmatch,{error,timeout}}},
|
{trace,
|
[{goxdcr_rest,send_with_timeout,5,
|
[{file,"src/goxdcr_rest.erl"},{line,54}]},
|
{goxdcr_rest,query_goxdcr,4,
|
[{file,"src/goxdcr_rest.erl"},{line,112}]},
|
{goxdcr_rest,get_from_goxdcr,3,
|
[{file,"src/goxdcr_rest.erl"},{line,135}]},
|
{ns_doctor,build_tasks_list,2,
|
[{file,"src/ns_doctor.erl"},{line,222}]},
|
{menelaus_web_misc,do_handle_tasks,3,
|
[{file,"src/menelaus_web_misc.erl"},
|
{line,63}]},
|
{request_throttler,do_request,3,
|
[{file,"src/request_throttler.erl"},
|
{line,58}]},
|
{menelaus_util,handle_request,2,
|
[{file,"src/menelaus_util.erl"},
|
{line,218}]},
|
{mochiweb_http,headers,6,
|
[{file,
|
"/home/couchbase/jenkins/workspace/couchbase-server-unix/couchdb/src/mochiweb/mochiweb_http.erl"},
|
{line,150}]}]}]
|
4. Hard Failover rebalance out of 172.23.104.157:8091. . Error log on 172.23.108.103 -
[ns_server:error,2021-10-12T10:21:15.101-07:00,ns_1@172.23.108.103:<0.1468.336>:menelaus_util:reply_server_error:207]Server error during processing: ["web request failed",
|
{path,"/pools/default/tasks"},
|
{method,'GET'},
|
{type,error},
|
{what,{badmatch,{error,timeout}}},
|
{trace,
|
[{goxdcr_rest,send_with_timeout,5,
|
[{file,"src/goxdcr_rest.erl"},{line,54}]},
|
{goxdcr_rest,query_goxdcr,4,
|
[{file,"src/goxdcr_rest.erl"},{line,112}]},
|
{goxdcr_rest,get_from_goxdcr,3,
|
[{file,"src/goxdcr_rest.erl"},{line,135}]},
|
{ns_doctor,build_tasks_list,2,
|
[{file,"src/ns_doctor.erl"},{line,222}]},
|
{menelaus_web_misc,do_handle_tasks,3,
|
[{file,"src/menelaus_web_misc.erl"},
|
{line,63}]},
|
{request_throttler,do_request,3,
|
[{file,"src/request_throttler.erl"},
|
{line,58}]},
|
{menelaus_util,handle_request,2,
|
[{file,"src/menelaus_util.erl"},
|
{line,218}]},
|
{mochiweb_http,headers,6,
|
[{file,
|
"/home/couchbase/jenkins/workspace/couchbase-server-unix/couchdb/src/mochiweb/mochiweb_http.erl"},
|
{line,150}]}]}]
|
5. Rebalance 2021-10-12T10:39:37. Error log on 172.23.108.103 -
[ns_server:error,2021-10-12T10:39:32.247-07:00,ns_1@172.23.108.103:<0.31339.339>:menelaus_util:reply_server_error:207]Server error during processing: ["web request failed",
|
{path,"/pools/default/tasks"},
|
{method,'GET'},
|
{type,error},
|
{what,{badmatch,{error,timeout}}},
|
{trace,
|
[{goxdcr_rest,send_with_timeout,5,
|
[{file,"src/goxdcr_rest.erl"},{line,54}]},
|
{goxdcr_rest,query_goxdcr,4,
|
[{file,"src/goxdcr_rest.erl"},{line,112}]},
|
{goxdcr_rest,get_from_goxdcr,3,
|
[{file,"src/goxdcr_rest.erl"},{line,135}]},
|
{ns_doctor,build_tasks_list,2,
|
[{file,"src/ns_doctor.erl"},{line,222}]},
|
{menelaus_web_misc,do_handle_tasks,3,
|
[{file,"src/menelaus_web_misc.erl"},
|
{line,63}]},
|
{request_throttler,do_request,3,
|
[{file,"src/request_throttler.erl"},
|
{line,58}]},
|
{menelaus_util,handle_request,2,
|
[{file,"src/menelaus_util.erl"},
|
{line,218}]},
|
{mochiweb_http,headers,6,
|
[{file,
|
"/home/couchbase/jenkins/workspace/couchbase-server-unix/couchdb/src/mochiweb/mochiweb_http.erl"},
|
{line,150}]}]}]
|
Attachments
Issue Links
- depends on
-
-
- Closed
-
-
-
- Closed
-
- duplicates
-
-
- Closed
-
- is caused by
-
MB-49057 XDCR - P2P - handle potential burst of incoming requests
-
- Closed
-
- relates to
-
-
- Closed
-
-
-
- Closed
-
-
-
- Closed
-