[System Test][Magma] Rebalance exited with reason bad_replicas.

-test tests/integration/neo/test_neo_kv_gsi_coll_xdcr_backup_sgw_fts_itemct_txns_eventing_cbas_scale3_magma.yml -scope tests/integration/neo/scope_neo_with_backup_magma.yml

2021-09-27T21:43:55.575-07:00, ns_orchestrator:0:info:message(ns_1@172.23.104.155) - Starting rebalance, KeepNodes = ['ns_1@172.23.104.137','ns_1@172.23.104.155',

                                 'ns_1@172.23.104.157','ns_1@172.23.104.5',

                                 'ns_1@172.23.104.67','ns_1@172.23.104.69',

                                 'ns_1@172.23.104.70','ns_1@172.23.106.188',

                                 'ns_1@172.23.108.103','ns_1@172.23.120.245',

                                 'ns_1@172.23.121.117','ns_1@172.23.121.3',

                                 'ns_1@172.23.123.27','ns_1@172.23.123.28',

                                 'ns_1@172.23.96.148','ns_1@172.23.96.251',

                                 'ns_1@172.23.96.252','ns_1@172.23.96.253',

                                 'ns_1@172.23.97.119','ns_1@172.23.97.121',

                                 'ns_1@172.23.97.122','ns_1@172.23.97.239',

                                 'ns_1@172.23.97.242','ns_1@172.23.98.135',

                                 'ns_1@172.23.99.11','ns_1@172.23.99.20',

                                 'ns_1@172.23.99.21','ns_1@172.23.99.25'], EjectNodes = [], Failed over and being ejected nodes = ['ns_1@172.23.106.100']; no delta recovery nodes; Operation Id = 2f54a8733a0470cc666d91583e3b0eb8

2021-09-27T22:17:55.221-07:00, ns_rebalancer:2:info:message(ns_1@172.23.104.155) - Bad replicators after rebalance:

Missing = [{'ns_1@172.23.104.5','ns_1@172.23.96.148',18},

           {'ns_1@172.23.104.5','ns_1@172.23.96.148',19},

           {'ns_1@172.23.104.5','ns_1@172.23.96.148',20},

           {'ns_1@172.23.104.5','ns_1@172.23.96.148',21},

           {'ns_1@172.23.104.5','ns_1@172.23.96.148',22},

           {'ns_1@172.23.104.5','ns_1@172.23.96.148',23},

           {'ns_1@172.23.104.5','ns_1@172.23.96.148',24},

           {'ns_1@172.23.104.5','ns_1@172.23.96.148',25},

           {'ns_1@172.23.104.5','ns_1@172.23.96.148',26},

           {'ns_1@172.23.104.5','ns_1@172.23.96.148',62},

           {'ns_1@172.23.104.5','ns_1@172.23.96.148',63},

           {'ns_1@172.23.104.5','ns_1@172.23.96.148',94},

           {'ns_1@172.23.108.103','ns_1@172.23.96.148',87},

           {'ns_1@172.23.108.103','ns_1@172.23.96.148',181},

           {'ns_1@172.23.108.103','ns_1@172.23.96.148',182},

           {'ns_1@172.23.108.103','ns_1@172.23.96.148',183},

           {'ns_1@172.23.108.103','ns_1@172.23.96.148',184},

           {'ns_1@172.23.108.103','ns_1@172.23.96.148',185},

           {'ns_1@172.23.108.103','ns_1@172.23.96.148',186},

           {'ns_1@172.23.108.103','ns_1@172.23.96.148',187},

           {'ns_1@172.23.108.103','ns_1@172.23.96.148',188},

           {'ns_1@172.23.108.103','ns_1@172.23.96.148',189},

           {'ns_1@172.23.108.103','ns_1@172.23.96.148',224},

           {'ns_1@172.23.108.103','ns_1@172.23.96.148',225},

           {'ns_1@172.23.97.119','ns_1@172.23.96.148',105},

           {'ns_1@172.23.97.119','ns_1@172.23.96.148',106},

           {'ns_1@172.23.97.119','ns_1@172.23.96.148',107},

           {'ns_1@172.23.97.119','ns_1@172.23.96.148',108},

           {'ns_1@172.23.97.119','ns_1@172.23.96.148',109},

           {'ns_1@172.23.97.119','ns_1@172.23.96.148',110},

           {'ns_1@172.23.97.119','ns_1@172.23.96.148',111},

           {'ns_1@172.23.97.119','ns_1@172.23.96.148',112},

           {'ns_1@172.23.97.119','ns_1@172.23.96.148',357},

           {'ns_1@172.23.97.119','ns_1@172.23.96.148',358},

           {'ns_1@172.23.97.119','ns_1@172.23.96.148',402},

           {'ns_1@172.23.97.119','ns_1@172.23.96.148',403},

           {'ns_1@172.23.97.121','ns_1@172.23.96.148',455},

           {'ns_1@172.23.97.121','ns_1@172.23.96.148',456},

           {'ns_1@172.23.97.121','ns_1@172.23.96.148',457},

           {'ns_1@172.23.97.121','ns_1@172.23.96.148',458},

           {'ns_1@172.23.97.121','ns_1@172.23.96.148',459},

           {'ns_1@172.23.97.121','ns_1@172.23.96.148',460},

           {'ns_1@172.23.97.121','ns_1@172.23.96.148',461},

           {'ns_1@172.23.97.121','ns_1@172.23.96.148',462},

           {'ns_1@172.23.97.121','ns_1@172.23.96.148',463},

           {'ns_1@172.23.97.121','ns_1@172.23.96.148',488},

           {'ns_1@172.23.97.121','ns_1@172.23.96.148',489},

           {'ns_1@172.23.97.121','ns_1@172.23.96.148',724},

           {'ns_1@172.23.97.122','ns_1@172.23.96.148',123},

           {'ns_1@172.23.97.122','ns_1@172.23.96.148',539},

           {'ns_1@172.23.97.122','ns_1@172.23.96.148',540},

           {'ns_1@172.23.97.122','ns_1@172.23.96.148',541},

           {'ns_1@172.23.97.122','ns_1@172.23.96.148',542},

           {'ns_1@172.23.97.122','ns_1@172.23.96.148',543},

           {'ns_1@172.23.97.122','ns_1@172.23.96.148',544},

           {'ns_1@172.23.97.122','ns_1@172.23.96.148',545},

           {'ns_1@172.23.97.122','ns_1@172.23.96.148',546},

           {'ns_1@172.23.97.122','ns_1@172.23.96.148',570},

           {'ns_1@172.23.97.122','ns_1@172.23.96.148',571},

           {'ns_1@172.23.97.239','ns_1@172.23.96.148',623},

           {'ns_1@172.23.97.239','ns_1@172.23.96.148',624},

           {'ns_1@172.23.97.239','ns_1@172.23.96.148',625},

           {'ns_1@172.23.97.239','ns_1@172.23.96.148',626},

           {'ns_1@172.23.97.239','ns_1@172.23.96.148',627},

           {'ns_1@172.23.97.239','ns_1@172.23.96.148',628},

           {'ns_1@172.23.97.239','ns_1@172.23.96.148',629},

           {'ns_1@172.23.97.239','ns_1@172.23.96.148',630},

           {'ns_1@172.23.97.239','ns_1@172.23.96.148',631},

           {'ns_1@172.23.97.239','ns_1@172.23.96.148',652},

           {'ns_1@172.23.97.239','ns_1@172.23.96.148',653},

           {'ns_1@172.23.99.20','ns_1@172.23.96.148',147},

           {'ns_1@172.23.99.20','ns_1@172.23.96.148',742},

           {'ns_1@172.23.99.20','ns_1@172.23.96.148',743},

           {'ns_1@172.23.99.20','ns_1@172.23.96.148',794},

           {'ns_1@172.23.99.20','ns_1@172.23.96.148',795},

           {'ns_1@172.23.99.20','ns_1@172.23.96.148',796},

           {'ns_1@172.23.99.20','ns_1@172.23.96.148',797},

           {'ns_1@172.23.99.20','ns_1@172.23.96.148',798},

           {'ns_1@172.23.99.20','ns_1@172.23.96.148',799},

           {'ns_1@172.23.99.20','ns_1@172.23.96.148',800},

           {'ns_1@172.23.99.20','ns_1@172.23.96.148',801},

           {'ns_1@172.23.99.21','ns_1@172.23.96.148',157},

           {'ns_1@172.23.99.21','ns_1@172.23.96.148',755},

           {'ns_1@172.23.99.21','ns_1@172.23.96.148',756},

           {'ns_1@172.23.99.21','ns_1@172.23.96.148',880},

           {'ns_1@172.23.99.21','ns_1@172.23.96.148',881},

           {'ns_1@172.23.99.21','ns_1@172.23.96.148',882},

           {'ns_1@172.23.99.21','ns_1@172.23.96.148',883},

           {'ns_1@172.23.99.21','ns_1@172.23.96.148',884},

           {'ns_1@172.23.99.21','ns_1@172.23.96.148',885},

           {'ns_1@172.23.99.21','ns_1@172.23.96.148',886},

           {'ns_1@172.23.99.21','ns_1@172.23.96.148',887},

           {'ns_1@172.23.99.25','ns_1@172.23.96.148',164},

           {'ns_1@172.23.99.25','ns_1@172.23.96.148',764},

           {'ns_1@172.23.99.25','ns_1@172.23.96.148',765},

           {'ns_1@172.23.99.25','ns_1@172.23.96.148',966},

           {'ns_1@172.23.99.25','ns_1@172.23.96.148',967},

           {'ns_1@172.23.99.25','ns_1@172.23.96.148',968},

           {'ns_1@172.23.99.25','ns_1@172.23.96.148',969},

           {'ns_1@172.23.99.25','ns_1@172.23.96.148',970},

           {'ns_1@172.23.99.25','ns_1@172.23.96.148',971},

           {'ns_1@172.23.99.25','ns_1@172.23.96.148',972},

           {'ns_1@172.23.99.25','ns_1@172.23.96.148',973}]

Extras = []

2021-09-27T22:17:55.224-07:00, ns_orchestrator:0:critical:message(ns_1@172.23.104.155) - Rebalance exited with reason bad_replicas.

Rebalance Operation Id = 2f54a8733a0470cc666d91583e3b0eb8