Details
-
Bug
-
Resolution: Fixed
-
Critical
-
7.6.0
-
7.6.0-2052
-
Triaged
-
0
-
Unknown
Description
There have been multiple failures. I have ignored the first 2 failures ( 1 was test induced and 1 was also expected because of https://issues.couchbase.com/browse/MB-60534). But the next few ones need to be analyzed -
Seen on iteration 13 -
failure 1 |
|
|
[user:error,2024-01-25T12:23:00.532-08:00,ns_1@172.23.97.67:<0.22016.0>:ns_orchestrator:log_rebalance_completion:1661]Rebalance exited with reason {service_rebalance_failed,index, |
{worker_died,
|
{'EXIT',<0.978.204>, |
{task_failed,rebalance,inactivity_timeout}}}}.
|
failure 2 |
[user:error,2024-01-25T12:57:00.644-08:00,ns_1@172.23.97.67:<0.22016.0>:ns_orchestrator:log_rebalance_completion:1661]Rebalance exited with reason {service_rebalance_failed,index, |
{{badmatch,
|
{error,
|
{bad_nodes,index,set_service_manager,
|
[{'ns_1@172.23.97.109', |
{exit,
|
{{linked_process_died,<34854.16026.124>, |
{'ns_1@172.23.97.109', |
{no_connection,"index-service_api"}}}, |
{gen_server,call,
|
[{'service_agent-index', |
'ns_1@172.23.97.109'}, |
{set_service_manager,<0.25785.215>}, |
infinity]}}}},
|
{'ns_1@172.23.97.108', |
{exit,
|
{{linked_process_died,<34853.11467.128>, |
{'ns_1@172.23.97.108', |
{no_connection,"index-service_api"}}}, |
{gen_server,call,
|
[{'service_agent-index', |
'ns_1@172.23.97.108'}, |
{set_service_manager,<0.25785.215>}, |
infinity]}}}}]}}},
|
[{service_manager,set_service_manager,1, |
[{file,"src/service_manager.erl"}, |
{line,188}]}, |
{service_manager,run_op,1, |
[{file,"src/service_manager.erl"}, |
{line,146}]}, |
{proc_lib,init_p,3, |
[{file,"proc_lib.erl"},{line,225}]}]}}. |
cbcollect ->
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1706216054/collectinfo-2024-01-25T210236-ns_1%40172.23.106.176.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1706216054/collectinfo-2024-01-25T210236-ns_1%40172.23.106.30.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1706216054/collectinfo-2024-01-25T210236-ns_1%40172.23.96.198.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1706216054/collectinfo-2024-01-25T210236-ns_1%40172.23.96.230.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1706216054/collectinfo-2024-01-25T210236-ns_1%40172.23.96.245.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1706216054/collectinfo-2024-01-25T210236-ns_1%40172.23.97.100.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1706216054/collectinfo-2024-01-25T210236-ns_1%40172.23.97.108.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1706216054/collectinfo-2024-01-25T210236-ns_1%40172.23.97.109.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1706216054/collectinfo-2024-01-25T210236-ns_1%40172.23.97.66.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1706216054/collectinfo-2024-01-25T210236-ns_1%40172.23.97.67.zip
Seen on iteration 14
[user:error,2024-01-25T13:07:00.674-08:00,ns_1@172.23.97.67:<0.22016.0>:ns_orchestrator:log_rebalance_completion:1661]Rebalance exited with reason {service_rebalance_failed,index, |
{{badmatch,
|
{error,
|
{bad_nodes,index,set_service_manager,
|
[{'ns_1@172.23.97.108', |
{exit,
|
{{linked_process_died,<34853.25107.129>, |
{'ns_1@172.23.97.108', |
{no_connection,"index-service_api"}}}, |
{gen_server,call,
|
[{'service_agent-index', |
'ns_1@172.23.97.108'}, |
{set_service_manager,<0.18468.218>}, |
infinity]}}}},
|
{'ns_1@172.23.97.109', |
{exit,
|
{{linked_process_died,<34854.28756.125>, |
{'ns_1@172.23.97.109', |
{no_connection,"index-service_api"}}}, |
{gen_server,call,
|
[{'service_agent-index', |
'ns_1@172.23.97.109'}, |
{set_service_manager,<0.18468.218>}, |
infinity]}}}}]}}},
|
[{service_manager,set_service_manager,1, |
[{file,"src/service_manager.erl"}, |
{line,188}]}, |
{service_manager,run_op,1, |
[{file,"src/service_manager.erl"}, |
{line,146}]}, |
{proc_lib,init_p,3, |
[{file,"proc_lib.erl"},{line,225}]}]}}. |
Rebalance Operation Id = 354686cd4a410959ca39935ca77119ee
|
|
|
[user:error,2024-01-25T13:23:00.722-08:00,ns_1@172.23.97.67:<0.22016.0>:ns_orchestrator:log_rebalance_completion:1661]Rebalance exited with reason {service_rebalance_failed,index, |
{{badmatch,
|
{error,
|
{bad_nodes,index,set_service_manager,
|
[{'ns_1@172.23.97.108', |
{exit,
|
{{linked_process_died,<34853.8768.132>, |
{'ns_1@172.23.97.108', |
{no_connection,"index-service_api"}}}, |
{gen_server,call,
|
[{'service_agent-index', |
'ns_1@172.23.97.108'}, |
{set_service_manager,<0.23376.224>}, |
infinity]}}}},
|
{'ns_1@172.23.97.109', |
{exit,
|
{{linked_process_died,<34854.13909.128>, |
{'ns_1@172.23.97.109', |
{no_connection,"index-service_api"}}}, |
{gen_server,call,
|
[{'service_agent-index', |
'ns_1@172.23.97.109'}, |
{set_service_manager,<0.23376.224>}, |
infinity]}}}}]}}},
|
[{service_manager,set_service_manager,1, |
[{file,"src/service_manager.erl"}, |
{line,188}]}, |
{service_manager,run_op,1, |
[{file,"src/service_manager.erl"}, |
{line,146}]}, |
{proc_lib,init_p,3, |
[{file,"proc_lib.erl"},{line,225}]}]}}. |
Rebalance Operation Id = 24549d086c17276f9913d5abba85412a
|
|
|
[user:error,2024-01-25T14:08:00.921-08:00,ns_1@172.23.97.67:<0.22016.0>:ns_orchestrator:log_rebalance_completion:1661]Rebalance exited with reason {service_rebalance_failed,index, |
{{badmatch,
|
{error,
|
{bad_nodes,index,set_service_manager,
|
[{'ns_1@172.23.97.108', |
{exit,
|
{{linked_process_died,<34853.5833.139>, |
{'ns_1@172.23.97.108', |
{no_connection,"index-service_api"}}}, |
{gen_server,call,
|
[{'service_agent-index', |
'ns_1@172.23.97.108'}, |
{set_service_manager,<0.19874.238>}, |
infinity]}}}},
|
{'ns_1@172.23.97.109', |
{exit,
|
{{linked_process_died,<34854.11461.135>, |
{'ns_1@172.23.97.109', |
{no_connection,"index-service_api"}}}, |
{gen_server,call,
|
[{'service_agent-index', |
'ns_1@172.23.97.109'}, |
{set_service_manager,<0.19874.238>}, |
infinity]}}}}]}}},
|
[{service_manager,set_service_manager,1, |
[{file,"src/service_manager.erl"}, |
{line,188}]}, |
{service_manager,run_op,1, |
[{file,"src/service_manager.erl"}, |
{line,146}]}, |
{proc_lib,init_p,3, |
[{file,"proc_lib.erl"},{line,225}]}]}}. |
Rebalance Operation Id = 91331030ddb57ec7fe31881a26330360
|
|
|
[user:error,2024-01-25T14:17:00.959-08:00,ns_1@172.23.97.67:<0.22016.0>:ns_orchestrator:log_rebalance_completion:1661]Rebalance exited with reason {service_rebalance_failed,index, |
{{badmatch,
|
{error,
|
{bad_nodes,index,set_service_manager,
|
[{'ns_1@172.23.97.108', |
{exit,
|
{{linked_process_died,<34853.15789.140>, |
{'ns_1@172.23.97.108', |
{no_connection,"index-service_api"}}}, |
{gen_server,call,
|
[{'service_agent-index', |
'ns_1@172.23.97.108'}, |
{set_service_manager,<0.8120.241>}, |
infinity]}}}},
|
{'ns_1@172.23.97.109', |
{exit,
|
{{linked_process_died,<34854.23414.136>, |
{'ns_1@172.23.97.109', |
{no_connection,"index-service_api"}}}, |
{gen_server,call,
|
[{'service_agent-index', |
'ns_1@172.23.97.109'}, |
{set_service_manager,<0.8120.241>}, |
infinity]}}}}]}}},
|
[{service_manager,set_service_manager,1, |
[{file,"src/service_manager.erl"}, |
{line,188}]}, |
{service_manager,run_op,1, |
[{file,"src/service_manager.erl"}, |
{line,146}]}, |
{proc_lib,init_p,3, |
[{file,"proc_lib.erl"},{line,225}]}]}}. |
Rebalance Operation Id = eb4dd71ee15f8a58d508988fdbba98f1
|
cbcollect ->
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1706220156/collectinfo-2024-01-25T221108-ns_1%40172.23.105.122.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1706220156/collectinfo-2024-01-25T221108-ns_1%40172.23.106.176.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1706220156/collectinfo-2024-01-25T221108-ns_1%40172.23.106.30.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1706220156/collectinfo-2024-01-25T221108-ns_1%40172.23.96.198.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1706220156/collectinfo-2024-01-25T221108-ns_1%40172.23.96.230.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1706220156/collectinfo-2024-01-25T221108-ns_1%40172.23.96.245.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1706220156/collectinfo-2024-01-25T221108-ns_1%40172.23.97.100.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1706220156/collectinfo-2024-01-25T221108-ns_1%40172.23.97.108.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1706220156/collectinfo-2024-01-25T221108-ns_1%40172.23.97.109.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1706220156/collectinfo-2024-01-25T221108-ns_1%40172.23.97.66.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1706220156/collectinfo-2024-01-25T221108-ns_1%40172.23.97.67.zip