[System Test] Rebalance failure - Timeout: Unable to evict enough memory

user:error,2023-10-27T09:50:18.438-07:00,ns_1@172.23.97.67<0.21947.0>:ns_orchestrator:log_rebalance_completion:1661]Rebalance exited with reason {service_rebalance_failed,index,

{"stageInfo":{"index":{"totalProgress":70.28868112070623,"perNodeProgress":{"ns_1@172.23.106.30":0.7028868112070623,"ns_1@172.23.106.171":0.7028868112070623,"ns_1@172.23.105.122":0.7028868112070623,"ns_1@172.23.106.176":0.7028868112070623,"ns_1@172.23.97.108":0.7028868112070623,"ns_1@172.23.97.109":0.7028868112070623},"startTime":"2023-10-27T09:19:25.084-07:00","completedTime":false,"timeTaken":1853379},"data":{"totalProgress":100,"perNodeProgress":{"ns_1@172.23.96.230":1,"ns_1@172.23.96.245":1,"ns_1@172.23.97.66":1,"ns_1@172.23.96.198":1},"startTime":"2023-10-27T09:19:20.580-07:00","completedTime":"2023-10-27T09:19:25.074-07:00","timeTaken":4494},"query":{"totalProgress":100,"perNodeProgress":{"ns_1@172.23.97.100":1,"ns_1@172.23.97.67":1},"startTime":"2023-10-27T09:19:25.074-07:00","completedTime":"2023-10-27T09:19:25.084-07:00","timeTaken":10}},"rebalanceId":"7992ba72ce8116e8cbe6d601542a0b52","nodesInfo":{"active_nodes":["ns_1@172.23.105.122","ns_1@172.23.106.171","ns_1@172.23.106.176","ns_1@172.23.106.30","ns_1@172.23.96.198","ns_1@172.23.96.230","ns_1@172.23.96.245","ns_1@172.23.97.100","ns_1@172.23.97.108","ns_1@172.23.97.109","ns_1@172.23.97.66","ns_1@172.23.97.67"],"keep_nodes":["ns_1@172.23.105.122","ns_1@172.23.106.171","ns_1@172.23.106.176","ns_1@172.23.106.30","ns_1@172.23.96.198","ns_1@172.23.96.230","ns_1@172.23.96.245","ns_1@172.23.97.100","ns_1@172.23.97.108","ns_1@172.23.97.109","ns_1@172.23.97.66","ns_1@172.23.97.67"],"eject_nodes":[],"delta_nodes":[],"failed_nodes":[]},"masterNode":"ns_1@172.23.97.67","startTime":"2023-10-27T09:19:20.567-07:00","completedTime":"2023-10-27T09:50:18.463-07:00","timeTaken":1857896,"completionMessage":"Rebalance exited with reason {service_rebalance_failed,index,\n                              {worker_died,\n                               {'EXIT',<0.13204.139>,\n                                {task_failed,rebalance,\n                                 {service_error,\n                                  <<\"Error creating slice Unable to initialize /data/@2i/bucket2_idx3_YjBhcfxt_12198899600882219760_3.index/docIndex, err = Timeout: Unable to evict enough memory 6208190100\">>}}}}}."}