Rebalance stuck for 40+ hrs during System Test

./sequoia -client 172.23.97.180:2375 -provider file:centos_second_cluster.yml -test tests/integration/7.2/test_7.2.yml -scope tests/integration/7.2/scope_7.2_magma.yml -scale 1 -repeat 0 -log_level 0 -version 7.2.6-8112 -skip_setup=false -skip_test=false -skip_teardown=true -skip_cleanup=false -continue=false -collect_on_error=false -stop_on_error=false -duration=604800 -show_topology=true

[user:info,2024-08-20T19:32:44.799-07:00,ns_1@172.23.218.153:<0.11428.0>:ns_orchestrator:idle:782]Starting rebalance, KeepNodes = ['ns_1@172.23.120.130','ns_1@172.23.120.132',

                                 'ns_1@172.23.120.135','ns_1@172.23.218.148',

                                 'ns_1@172.23.218.150','ns_1@172.23.218.151',

                                 'ns_1@172.23.218.152','ns_1@172.23.218.153',

                                 'ns_1@172.23.218.155','ns_1@172.23.218.156',

                                 'ns_1@172.23.218.157','ns_1@172.23.218.158',

                                 'ns_1@172.23.218.159','ns_1@172.23.218.160',

                                 'ns_1@172.23.218.161','ns_1@172.23.218.162',

                                 'ns_1@172.23.218.164','ns_1@172.23.218.181',

                                 'ns_1@172.23.218.182','ns_1@172.23.218.183',

                                 'ns_1@172.23.218.184','ns_1@172.23.218.185',

                                 'ns_1@172.23.218.186'], EjectNodes = ['ns_1@172.23.218.149'], Failed over and being ejected nodes = []; no delta recovery nodes; Operation Id = dabeca25ba76fe146832953e1efb80c0

[rebalance:info,2024-08-20T19:32:44.803-07:00,ns_1@172.23.218.153:<0.1059.678>:ns_rebalancer:drop_old_2i_indexes:1244]Going to drop possible old 2i indexes on nodes []

[rebalance:info,2024-08-20T19:32:44.803-07:00,ns_1@172.23.218.153:<0.1059.678>:ns_rebalancer:drop_old_2i_indexes:1250]Going to keep possible 2i indexes on nodes []

[user:info,2024-08-20T19:32:45.379-07:00,ns_1@172.23.218.153:<0.1059.678>:ns_rebalancer:rebalance_bucket:591]Started rebalancing bucket ITEM

[rebalance:info,2024-08-20T19:32:45.385-07:00,ns_1@172.23.218.153:<0.1059.678>:ns_rebalancer:rebalance_bucket:592]Rebalancing bucket "ITEM" with config [{deltaRecoveryMap,undefined},

http://172.23.218.161:8091/pools/default/rebalanceProgress

{"status":"running","ns_1@172.23.218.160":{"progress":0.7962962962962963},"ns_1@172.23.120.130":{"progress":0.7699999999999999},"ns_1@172.23.218.150":{"progress":0},"ns_1@172.23.218.151":{"progress":0},"ns_1@172.23.218.161":{"progress":0},"ns_1@172.23.218.181":{"progress":0},"ns_1@172.23.218.162":{"progress":0},"ns_1@172.23.218.152":{"progress":0.7999999999999999},"ns_1@172.23.120.132":{"progress":0.7928571428571428},"ns_1@172.23.218.182":{"progress":0},"ns_1@172.23.218.183":{"progress":0.7814814814814814},"ns_1@172.23.218.153":{"progress":0.7999999999999999},"ns_1@172.23.218.184":{"progress":0},"ns_1@172.23.218.164":{"progress":0},"ns_1@172.23.218.185":{"progress":0},"ns_1@172.23.218.155":{"progress":0.789655172413793},"ns_1@172.23.120.135":{"progress":0.7928571428571428},"ns_1@172.23.218.186":{"progress":0},"ns_1@172.23.218.156":{"progress":0.7925925925925925},"ns_1@172.23.218.157":{"progress":0},"ns_1@172.23.218.158":{"progress":0},"ns_1@172.23.218.148":{"progress":0},"ns_1@172.23.218.149":{"progress":0.7670212765957446},"ns_1@172.23.218.159":{"progress":0.774074074074074}}