Details
-
Bug
-
Resolution: Duplicate
-
Critical
-
7.1.0
-
Untriaged
-
-
1
-
Unknown
Description
Build : 7.1.0-2126
Test : -test tests/integration/neo/test_neo_couchstore_milestone4.yml -scope tests/integration/neo/scope_couchstore.yml
Scale : 3
Iteration : 7th (Day 4/5)
In the longevity test, there is a step to perform an auto-failover for an indexer node 172.23.99.11. A rebalance operation follows the auto failover. This rebalance operation is in progress since 23+ hrs. Upon checking the indexes in Moving state, it appears that the indexes are being moved and getting built, but very slowly.
Not sure if this issue is related to MB-50431. Please close this as a duplicate if it is.
Attached are the latest logs. If you need the logs from an older timestamp, please let me know.
Indexer nodes currently in the cluster : 172.23.104.137, 172.23.104.155, 172.23.104.157, 172.23.120.245, 172.23.96.251, 172.23.99.25
The current set of indexes in Moving state are :
{
|
"bucket" : "bucket4",
|
"collection" : "coll_1",
|
"completion" : 87,
|
"definition" : "CREATE INDEX `idx6_oZkOl1` ON `bucket4`.`scope_5`.`coll_1`(`city`) PARTITION BY hash((meta().`id`)) WITH { \"nodes\":[ \"172.23.104.137:8091\",\"172.23.104.155:8091\",\"172.23.104.157:8091\",\"172.23.120.245:8091\",\"172.23.96.251:8091\" ], \"num_replica\":3, \"num_partition\":4 }",
|
"defnId" : 9134863151071151210,
|
"hosts" : [
|
"172.23.104.137:8091",
|
"172.23.104.157:8091",
|
"172.23.120.245:8091",
|
"172.23.96.251:8091"
|
],
|
"indexName" : "idx6_oZkOl1",
|
"indexType" : "plasma",
|
"instId" : 18040294700613587003,
|
"lastScanTime" : "NA",
|
"name" : "idx6_oZkOl1 (replica 3)",
|
"numPartition" : 4,
|
"numReplica" : 3,
|
"partitionMap" : {
|
"172.23.104.137:8091" : [
|
1
|
],
|
"172.23.104.157:8091" : [
|
2
|
],
|
"172.23.120.245:8091" : [
|
3
|
],
|
"172.23.96.251:8091" : [
|
4
|
]
|
},
|
"partitioned" : true,
|
"progress" : 87.5162447750164,
|
"replicaId" : 3,
|
"scheduled" : false,
|
"scope" : "scope_5",
|
"secExprs" : [
|
"`city`"
|
],
|
"stale" : false,
|
"status" : "Moving"
|
}
|
{
|
"bucket" : "bucket5",
|
"collection" : "coll_4",
|
"completion" : 0,
|
"definition" : "CREATE INDEX `idx7_aIwQ` ON `bucket5`.`scope_2`.`coll_4`(`price`,`name`,`city`,`country`) PARTITION BY hash((meta().`id`)) WITH { \"nodes\":[ \"172.23.104.137:8091\",\"172.23.104.155:8091\",\"172.23.104.157:8091\",\"172.23.120.245:8091\",\"172.23.96.251:8091\" ], \"num_replica\":2, \"num_partition\":4 }",
|
"defnId" : 5034203126402128860,
|
"hosts" : [
|
"172.23.104.137:8091"
|
],
|
"indexName" : "idx7_aIwQ",
|
"indexType" : "plasma",
|
"instId" : 6742786560512970384,
|
"lastScanTime" : "NA",
|
"name" : "idx7_aIwQ (replica 1)",
|
"numPartition" : 1,
|
"numReplica" : 2,
|
"partitionMap" : {
|
"172.23.104.137:8091" : [
|
3
|
]
|
},
|
"partitioned" : true,
|
"progress" : 0.118475100726513,
|
"replicaId" : 1,
|
"scheduled" : false,
|
"scope" : "scope_2",
|
"secExprs" : [
|
"`price`",
|
"`name`",
|
"`city`",
|
"`country`"
|
],
|
"stale" : false,
|
"status" : "Moving"
|
}
|
{
|
"bucket" : "bucket5",
|
"collection" : "coll_4",
|
"completion" : 100,
|
"definition" : "CREATE INDEX `idx9_i82sqkSS` ON `bucket5`.`scope_2`.`coll_4`((all (array flatten_keys((`r`.`author`), ((`r`.`ratings`).`Rooms`)) for `r` in `reviews` end)),`free_parking`) PARTITION BY hash((meta().`id`)) WITH { \"defer_build\":true, \"nodes\":[ \"172.23.104.137:8091\",\"172.23.104.155:8091\",\"172.23.104.157:8091\",\"172.23.120.245:8091\",\"172.23.96.251:8091\" ], \"num_replica\":2, \"num_partition\":4 }",
|
"defnId" : 5938134265000410345,
|
"hosts" : [
|
"172.23.104.137:8091"
|
],
|
"indexName" : "idx9_i82sqkSS",
|
"indexType" : "plasma",
|
"instId" : 16836682486729332750,
|
"lastScanTime" : "Sun Jan 30 13:26:53 PST 2022",
|
"name" : "idx9_i82sqkSS (replica 2)",
|
"numPartition" : 1,
|
"numReplica" : 2,
|
"partitionMap" : {
|
"172.23.104.137:8091" : [
|
2
|
]
|
},
|
"partitioned" : true,
|
"progress" : 0.118475100726513,
|
"replicaId" : 2,
|
"scheduled" : false,
|
"scope" : "scope_2",
|
"secExprs" : [
|
"(all (array flatten_keys((`r`.`author`), ((`r`.`ratings`).`Rooms`)) for `r` in `reviews` end))",
|
"`free_parking`"
|
],
|
"stale" : false,
|
"status" : "Moving"
|
}
|