Details
-
Bug
-
Resolution: Fixed
-
Critical
-
7.1.0
-
Untriaged
-
-
1
-
Unknown
Description
Build : 7.1.0-1846
Test :-test tests/integration/neo/test_neo_magma_milestone3.yml -scope tests/integration/neo/scope_neo_magma.yml (Magma)
Scale : 3
Iteration : 1st
Rebalance operation to add a new index node 172.23.106.54 to the cluster is hung for over 16 hrs now.
There are 2 indexes that are in Moving state :
{
|
"bucket" : "bucket6",
|
"collection" : "coll_3",
|
"completion" : 50,
|
"definition" : "CREATE INDEX `idx2_O7YGmypH` ON `bucket6`.`scope_1`.`coll_3`(`free_breakfast`,`type`,`free_parking`,array_count(`public_likes`),`price`,`country`) WITH { \"defer_build\":true, \"nodes\":[ \"172.23.104.249:8091\",\"172.23.105.38:8091\",\"172.23.105.39:8091\",\"172.23.108.140:8091\" ], \"num_replica\":2 }",
|
"defnId" : 13866751409711659627,
|
"hosts" : [
|
"172.23.105.38:8091",
|
"172.23.105.39:8091"
|
],
|
"indexName" : "idx2_O7YGmypH",
|
"indexType" : "plasma",
|
"instId" : 12374794132800418394,
|
"lastScanTime" : "Wed Dec 8 20:11:57 PST 2021",
|
"name" : "idx2_O7YGmypH (replica 1)",
|
"numPartition" : 2,
|
"numReplica" : 2,
|
"partitionMap" : {
|
"172.23.105.38:8091" : [
|
0
|
],
|
"172.23.105.39:8091" : [
|
0
|
]
|
},
|
"partitioned" : false,
|
"progress" : 50.0162815618221,
|
"replicaId" : 1,
|
"scheduled" : false,
|
"scope" : "scope_1",
|
"secExprs" : [
|
"`free_breakfast`",
|
"`type`",
|
"`free_parking`",
|
"array_count(`public_likes`)",
|
"`price`",
|
"`country`"
|
],
|
"stale" : false,
|
"status" : "Moving"
|
}
|
and
{
|
"bucket" : "N1QL_SYSTEM_BUCKET",
|
"collection" : "N1QL_CBO_STATS",
|
"completion" : 70,
|
"definition" : "CREATE PRIMARY INDEX `PRIMARY_IDX_CBO_STATS` ON `N1QL_SYSTEM_BUCKET`.`N1QL_SYSTEM_SCOPE`.`N1QL_CBO_STATS` WITH { \"nodes\":[ \"172.23.105.39:8091\",\"172.23.108.140:8091\" ] }",
|
"defnId" : 3685596331295969246,
|
"hosts" : [
|
"172.23.105.39:8091",
|
"172.23.108.140:8091"
|
],
|
"indexName" : "PRIMARY_IDX_CBO_STATS",
|
"indexType" : "plasma",
|
"instId" : 15149035466292205977,
|
"isPrimary" : true,
|
"lastScanTime" : "NA",
|
"name" : "PRIMARY_IDX_CBO_STATS",
|
"numPartition" : 2,
|
"numReplica" : 0,
|
"partitionMap" : {
|
"172.23.105.39:8091" : [
|
0
|
],
|
"172.23.108.140:8091" : [
|
0
|
]
|
},
|
"partitioned" : false,
|
"progress" : 70.7523245984776,
|
"replicaId" : 0,
|
"scheduled" : false,
|
"scope" : "N1QL_SYSTEM_SCOPE",
|
"stale" : false,
|
"status" : "Moving"
|
}
|
The number of pending mutations for PRIMARY_IDX_CBO_STATS index shown on the UI is 78972T ! While in the indexer_stats.log, the latest value for num_docs_pending is 4692. Let me know if this issue should be tracked separately.
2021-12-08T20:54:04.876-08:00 index_N1QL_SYSTEM_BUCKET:N1QL_SYSTEM_SCOPE:N1QL_CBO_STATS:PRIMARY_IDX_CBO_STATS:15149035466292205977
{"num_docs_pending":4692,"progress_stat_time":"1639025644657065003"}Indexer nodes : 172.23.104.249:8091 172.23.105.0:8091 172.23.105.38:8091 172.23.105.39:8091 172.23.108.132:8091 172.23.108.140:8091 172.23.106.54:8091