[System Test] Rebalance failures - copy failed :config.json copy failed /

[user:error,2024-02-07T05:05:43.592-08:00,ns_1@172.23.97.67:<0.10153.366>:ns_orchestrator:log_rebalance_completion:1661]Rebalance exited with reason {service_rebalance_failed,index,

                              {agent_died,<34855.11419.1620>,

                               {linked_process_died,<34855.13063.1620>,

                                {'ns_1@172.23.97.108',

                                 {timeout,

                                  {gen_server,call,

                                   [<34855.9403.1620>,

                                    {call,"ServiceAPI.GetTaskList",

                                     #Fun<json_rpc_connection.0.36915653>,

                                     #{timeout => 60000}},

                                    60000]}}}}}}.

Rebalance Operation Id = 96cf5475af598d1eb0d00b34e2bb1e38

[user:error,2024-02-07T05:08:09.104-08:00,ns_1@172.23.97.67:<0.10153.366>:ns_orchestrator:log_rebalance_completion:1661]Rebalance exited with reason {service_rebalance_failed,index,

                              {agent_died,<34859.13197.1364>,

                               {linked_process_died,<34859.26957.1567>,

                                {'ns_1@172.23.106.171',

                                 {timeout,

                                  {gen_server,call,

                                   [<34859.8648.1364>,

                                    {call,"ServiceAPI.StartTopologyChange",

                                     #Fun<json_rpc_connection.0.36915653>,

                                     #{timeout => 60000}},

                                    60000]}}}}}}.

[user:error,2024-02-07T07:37:46.598-08:00,ns_1@172.23.97.67:<0.10153.366>:ns_orchestrator:log_rebalance_completion:1661]Rebalance exited with reason {service_rebalance_failed,index,

                              {worker_died,

                               {'EXIT',<0.3945.1638>,

                                {task_failed,rebalance,

                                 {service_error,

                                  <<"shard copy aborted: LSS /data/@2i/shards/shard6124636750988113296/data metadata copy failed :config.json copy failed src :/data/@2i/shards/shard6124636750988113296/config.json dst :https://172.23.106.30:9104//plasma_storage_v1/7b1871d5533d3945a3bc73a93c51704e_ShardTokenbf_c7_1f_16_95_4_5c_a4/6124636750988113296/shards/shard6124636750988113296/config.json error :CopyBytes error : rpc remote file not open, shardId :6124636750988113296">>}}}}}.

Rebalance Operation Id = 82584f2818e09e8f0b2b39160324f562