Details
-
Bug
-
Resolution: Fixed
-
Critical
-
7.6.0
-
7.6.0-1704
-
Untriaged
-
0
-
Unknown
Description
The rebalance seems to have failed with this error
On node 172.23.97.67
user:error,2023-10-27T09:50:18.438-07:00,ns_1@172.23.97.67<0.21947.0>:ns_orchestrator:log_rebalance_completion:1661]Rebalance exited with reason {service_rebalance_failed,index, |
{"stageInfo":{"index":{"totalProgress":70.28868112070623,"perNodeProgress":{"ns_1@172.23.106.30":0.7028868112070623,"ns_1@172.23.106.171":0.7028868112070623,"ns_1@172.23.105.122":0.7028868112070623,"ns_1@172.23.106.176":0.7028868112070623,"ns_1@172.23.97.108":0.7028868112070623,"ns_1@172.23.97.109":0.7028868112070623},"startTime":"2023-10-27T09:19:25.084-07:00","completedTime":false,"timeTaken":1853379},"data":{"totalProgress":100,"perNodeProgress":{"ns_1@172.23.96.230":1,"ns_1@172.23.96.245":1,"ns_1@172.23.97.66":1,"ns_1@172.23.96.198":1},"startTime":"2023-10-27T09:19:20.580-07:00","completedTime":"2023-10-27T09:19:25.074-07:00","timeTaken":4494},"query":{"totalProgress":100,"perNodeProgress":{"ns_1@172.23.97.100":1,"ns_1@172.23.97.67":1},"startTime":"2023-10-27T09:19:25.074-07:00","completedTime":"2023-10-27T09:19:25.084-07:00","timeTaken":10}},"rebalanceId":"7992ba72ce8116e8cbe6d601542a0b52","nodesInfo":{"active_nodes":["ns_1@172.23.105.122","ns_1@172.23.106.171","ns_1@172.23.106.176","ns_1@172.23.106.30","ns_1@172.23.96.198","ns_1@172.23.96.230","ns_1@172.23.96.245","ns_1@172.23.97.100","ns_1@172.23.97.108","ns_1@172.23.97.109","ns_1@172.23.97.66","ns_1@172.23.97.67"],"keep_nodes":["ns_1@172.23.105.122","ns_1@172.23.106.171","ns_1@172.23.106.176","ns_1@172.23.106.30","ns_1@172.23.96.198","ns_1@172.23.96.230","ns_1@172.23.96.245","ns_1@172.23.97.100","ns_1@172.23.97.108","ns_1@172.23.97.109","ns_1@172.23.97.66","ns_1@172.23.97.67"],"eject_nodes":[],"delta_nodes":[],"failed_nodes":[]},"masterNode":"ns_1@172.23.97.67","startTime":"2023-10-27T09:19:20.567-07:00","completedTime":"2023-10-27T09:50:18.463-07:00","timeTaken":1857896,"completionMessage":"Rebalance exited with reason {service_rebalance_failed,index,\n {worker_died,\n {'EXIT',<0.13204.139>,\n {task_failed,rebalance,\n {service_error,\n <<\"Error creating slice Unable to initialize /data/@2i/bucket2_idx3_YjBhcfxt_12198899600882219760_3.index/docIndex, err = Timeout: Unable to evict enough memory 6208190100\">>}}}}}."} |
|
Cbcollect logs:
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1698425456/collectinfo-2023-10-27T170647-ns_1%40172.23.105.122.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1698425456/collectinfo-2023-10-27T170647-ns_1%40172.23.106.171.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1698425456/collectinfo-2023-10-27T170647-ns_1%40172.23.106.30.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1698425456/collectinfo-2023-10-27T170647-ns_1%40172.23.96.198.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1698425456/collectinfo-2023-10-27T170647-ns_1%40172.23.96.230.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1698425456/collectinfo-2023-10-27T170647-ns_1%40172.23.96.245.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1698425456/collectinfo-2023-10-27T170647-ns_1%40172.23.97.100.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1698425456/collectinfo-2023-10-27T170647-ns_1%40172.23.97.108.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1698425456/collectinfo-2023-10-27T170647-ns_1%40172.23.97.109.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1698425456/collectinfo-2023-10-27T170647-ns_1%40172.23.97.66.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1698425456/collectinfo-2023-10-27T170647-ns_1%40172.23.97.67.zip