[System Test] Rebalance failures - shard metadata mismatch with snapshot metadata

{"stageInfo":{"index":{"totalProgress":9.482071713147411,"perNodeProgress":{"ns_1@172.23.106.171":0.09482071713147411,"ns_1@172.23.105.122":0.09482071713147411,"ns_1@172.23.97.108":0.09482071713147411,"ns_1@172.23.97.109":0.09482071713147411},"startTime":"2023-12-01T11:59:08.912-08:00","completedTime":false,"timeTaken":56938},"data":{"totalProgress":100,"perNodeProgress":{"ns_1@172.23.96.230":1,"ns_1@172.23.96.245":1,"ns_1@172.23.97.66":1,"ns_1@172.23.96.198":1},"startTime":"2023-12-01T11:59:05.413-08:00","completedTime":"2023-12-01T11:59:08.903-08:00","timeTaken":3490},"query":{"totalProgress":100,"perNodeProgress":{"ns_1@172.23.97.100":1,"ns_1@172.23.97.67":1},"startTime":"2023-12-01T11:59:08.904-08:00","completedTime":"2023-12-01T11:59:08.912-08:00","timeTaken":8}},"rebalanceId":"5f610dd5a577a56a88dd6ec5e76f2ec6","nodesInfo":{"active_nodes":["ns_1@172.23.105.122","ns_1@172.23.106.171","ns_1@172.23.96.198","ns_1@172.23.96.230","ns_1@172.23.96.245","ns_1@172.23.97.100","ns_1@172.23.97.108","ns_1@172.23.97.109","ns_1@172.23.97.66","ns_1@172.23.97.67"],"keep_nodes":["ns_1@172.23.105.122","ns_1@172.23.106.171","ns_1@172.23.96.198","ns_1@172.23.96.230","ns_1@172.23.96.245","ns_1@172.23.97.100","ns_1@172.23.97.108","ns_1@172.23.97.109","ns_1@172.23.97.66","ns_1@172.23.97.67"],"eject_nodes":[],"delta_nodes":[],"failed_nodes":[]},"masterNode":"ns_1@172.23.97.67","startTime":"2023-12-01T11:59:02.537-08:00","completedTime":"2023-12-01T12:00:05.850-08:00","timeTaken":63313,"completionMessage":"Rebalance exited with reason {service_rebalance_failed,index,\n                              {worker_died,\n                               {'EXIT',<0.9297.170>,\n                                {task_failed,rebalance,\n                                 {service_error,\n                                  <<\"shard copy aborted: shard metadata mismatch with snapshot metadata, shardId :6111060446405480015\">>}}}}}."}