[System-test] Index rebalance failed with error "EquivIndexViolation"

Rebalance exited with reason {service_rebalance_failed,index,

                              {worker_died,

                               {'EXIT',<0.1540.153>,

                                {rebalance_failed,

                                 {service_error,

                                  <<"\nMemoryQuota: 58993934336\nCpuQuota: 36\n--- Violations for index <idx11_gGYNoR5Q 1 (replica 1), default8, scope_0, coll_8> (mem 179.297K, cpu 0) at node svc-i-node-018.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091 \n\tCannot move to svc-i-node-023.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: EquivIndexViolation (free mem 35.2258G, free cpu 30.57924303201806)\n\tCannot move to svc-i-node-021.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: ServerGroupViolation (free mem 43.6656G, free cpu 33.58705070115184)\n\tCannot move to svc-i-node-020.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: ReplicaViolation (free mem 41.0809G, free cpu 32.97896175774347)\n\tCannot move to svc-i-node-019.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: EquivIndexViolation (free mem 33.56G, free cpu 31.54383883791578)\n--- Violations for index <idx9_EGV0CTRWM3_idxprefix 2, default8, scope_0, coll_7> (mem 0, cpu 0) at node svc-i-node-018.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091 \n\tCannot move to svc-i-node-023.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: EquivIndexViolation (free mem 35.2258G, free cpu 30.57924303201806)\n\tCannot move to svc-i-node-021.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: ServerGroupViolation (free mem 43.6656G, free cpu 33.58705070115184)\n\tCannot move to svc-i-node-020.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: ReplicaViolation (free mem 41.0809G, free cpu 32.97896175774347)\n\tCannot move to svc-i-node-019.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: EquivIndexViolation (free mem 33.56G, free cpu 31.54383883791578)\n--- Violations for index <idx9_EGV0CTRWM3_idxprefix 4, default8, scope_0, coll_7> (mem 0, cpu 0) at node svc-i-node-018.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091 \n\tCannot move to svc-i-node-023.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: EquivIndexViolation (free mem 35.2258G, free cpu 30.57924303201806)\n\tCannot move to svc-i-node-021.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: ServerGroupViolation (free mem 43.6656G, free cpu 33.58705070115184)\n\tCannot move to svc-i-node-020.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: ReplicaViolation (free mem 41.0809G, free cpu 32.97896175774347)\n\tCannot move to svc-i-node-019.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: EquivIndexViolation (free mem 33.56G, free cpu 31.54383883791578)\n--- Violations for index <idx9_EGV0CTRWM3_idxprefix 3, default8, scope_0, coll_7> (mem 0, cpu 0) at node svc-i-node-018.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091 \n\tCannot move to svc-i-node-023.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: EquivIndexViolation (free mem 35.2258G, free cpu 30.57924303201806)\n\tCannot move to svc-i-node-021.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: ServerGroupViolation (free mem 43.6656G, free cpu 33.58705070115184)\n\tCannot move to svc-i-node-020.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: ReplicaViolation (free mem 41.0809G, free cpu 32.97896175774347)\n\tCannot move to svc-i-node-019.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: EquivIndexViolation (free mem 33.56G, free cpu 31.54383883791578)\n--- Violations for index <idx11_TKBCG4JS0O_idxprefix 2, default8, scope_0, coll_8> (mem 219.313K, cpu 0) at node svc-i-node-018.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091 \n\tCannot move to svc-i-node-023.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: EquivIndexViolation (free mem 35.2258G, free cpu 30.57924303201806)\n\tCannot move to svc-i-node-021.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: ServerGroupViolation (free mem 43.6656G, free cpu 33.58705070115184)\n\tCannot move to svc-i-node-020.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: ReplicaViolation (free mem 41.0809G, free cpu 32.97896175774347)\n\tCannot move to svc-i-node-019.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: EquivIndexViolation (free mem 33.56G, free cpu 31.54383883791578)\n--- Violations for index <idx11_TKBCG4JS0O_idxprefix 3 (replica 1), default8, scope_0, coll_8> (mem 66.2422K, cpu 0) at node svc-i-node-018.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091 \n\tCannot move to svc-i-node-023.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: EquivIndexViolation (free mem 35.2258G, free cpu 30.57924303201806)\n\tCannot move to svc-i-node-021.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: ServerGroupViolation (free mem 43.6656G, free cpu 33.58705070115184)\n\tCannot move to svc-i-node-020.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: ReplicaViolation (free mem 41.0809G, free cpu 32.97896175774347)\n\tCannot move to svc-i-node-019.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: EquivIndexViolation (free mem 33.56G, free cpu 31.54383883791578)\n--- Violations for index <idx7_DUQMJL2M21_idxprefix 1, default8, scope_0, coll_7> (mem 18.6533K, cpu 0) at node svc-i-node-018.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091 \n\tCannot move to svc-i-node-023.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: EquivIndexViolation (free mem 35.2258G, free cpu 30.57924303201806)\n\tCannot move to svc-i-node-021.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: ServerGroupViolation (free mem 43.6656G, free cpu 33.58705070115184)\n\tCannot move to svc-i-node-020.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: ReplicaViolation (free mem 41.0809G, free cpu 32.97896175774347)\n\tCannot move to svc-i-node-019.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: EquivIndexViolation (free mem 33.56G, free cpu 31.54383883791578)\n--- Violations for index <idx7_DUQMJL2M21_idxprefix 2 (replica 1), default8, scope_0, coll_7> (mem 18.2334K, cpu 0) at node svc-i-node-018.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091 \n\tCannot move to svc-i-node-023.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: EquivIndexViolation (free mem 35.2258G, free cpu 30.57924303201806)\n\tCannot move to svc-i-node-021.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: ServerGroupViolation (free mem 43.6656G, free cpu 33.58705070115184)\n\tCannot move to svc-i-node-020.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: ReplicaViolation (free mem 41.0809G, free cpu 32.97896175774347)\n\tCannot move to svc-i-node-019.fj0yg7wxliw5k7y9.sandbox.nonprod-project-avengers.com:18091: EquivIndexViolation (free mem 33.56G, free cpu 31.54383883791578)\n">>}}}}}.

Rebalance Operation Id = 78b52f295098dfb231faac5f00cd7acc

{

  "size": 5, 

  "services": ["index"],

  "aws": {

    "instanceSize": "c5.9xlarge",

    "ebsSizeGib": 400

  },

  "azure": {

    "instanceSize": "Standard_F16s_v2",

    "diskSize": "P50"

  },

  "gcp": {

    "instanceSize": "n2-custom-36-73728",

    "storageSizeGib": 450

  }

},