Details
-
Bug
-
Resolution: Fixed
-
Critical
-
7.2.1
-
None
-
Capella
-
Untriaged
-
-
0
-
Unknown
Description
During the testing of 7.2.1-5904 QE encountered a situation where a k/v rebalance got stuck during a scale out operation from 3 - 5 nodes.
Output of /pools/default/tasks at the time of the reported issue:
[
|
{
|
"statusId": "10150818059acce2964068ea2122a169", |
"type": "rebalance", |
"subtype": "rebalance", |
"recommendedRefreshPeriod": 0.25, |
"status": "running", |
"progress": 0, |
"perNode": { |
"ns_1@svc-dqisea-node-005.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com": { |
"progress": 0 |
},
|
"ns_1@svc-dqisea-node-004.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com": { |
"progress": 0 |
},
|
"ns_1@svc-dqisea-node-003.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com": { |
"progress": 0 |
},
|
"ns_1@svc-dqisea-node-002.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com": { |
"progress": 0 |
},
|
"ns_1@svc-dqisea-node-001.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com": { |
"progress": 0 |
}
|
},
|
"detailedProgress": { |
"bucket": "travel-sample", |
"bucketNumber": 1, |
"bucketsCount": 1, |
"perNode": { |
"ns_1@svc-dqisea-node-005.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com": { |
"ingoing": { |
"docsTotal": 3168517, |
"docsTransferred": 0, |
"activeVBucketsLeft": 205, |
"replicaVBucketsLeft": 205 |
},
|
"outgoing": { |
"docsTotal": 0, |
"docsTransferred": 0, |
"activeVBucketsLeft": 0, |
"replicaVBucketsLeft": 0 |
}
|
},
|
"ns_1@svc-dqisea-node-004.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com": { |
"ingoing": { |
"docsTotal": 3150359, |
"docsTransferred": 0, |
"activeVBucketsLeft": 205, |
"replicaVBucketsLeft": 203 |
},
|
"outgoing": { |
"docsTotal": 0, |
"docsTransferred": 0, |
"activeVBucketsLeft": 0, |
"replicaVBucketsLeft": 0 |
}
|
},
|
"ns_1@svc-dqisea-node-003.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com": { |
"ingoing": { |
"docsTotal": 405, |
"docsTransferred": 0, |
"activeVBucketsLeft": 0, |
"replicaVBucketsLeft": 0 |
},
|
"outgoing": { |
"docsTotal": 2349783, |
"docsTransferred": 0, |
"activeVBucketsLeft": 137, |
"replicaVBucketsLeft": 135 |
}
|
},
|
"ns_1@svc-dqisea-node-002.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com": { |
"ingoing": { |
"docsTotal": 46045, |
"docsTransferred": 0, |
"activeVBucketsLeft": 0, |
"replicaVBucketsLeft": 9 |
},
|
"outgoing": { |
"docsTotal": 2075306, |
"docsTransferred": 0, |
"activeVBucketsLeft": 136, |
"replicaVBucketsLeft": 145 |
}
|
},
|
"ns_1@svc-dqisea-node-001.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com": { |
"ingoing": { |
"docsTotal": 39256, |
"docsTransferred": 0, |
"activeVBucketsLeft": 0, |
"replicaVBucketsLeft": 11 |
},
|
"outgoing": { |
"docsTotal": 1979493, |
"docsTransferred": 0, |
"activeVBucketsLeft": 137, |
"replicaVBucketsLeft": 148 |
}
|
}
|
}
|
},
|
"stageInfo": { |
"analytics": { |
"startTime": false, |
"completedTime": false, |
"timeTaken": false |
},
|
"eventing": { |
"startTime": false, |
"completedTime": false, |
"timeTaken": false |
},
|
"search": { |
"startTime": false, |
"completedTime": false, |
"timeTaken": false |
},
|
"index": { |
"startTime": false, |
"completedTime": false, |
"timeTaken": false |
},
|
"data": { |
"startTime": "2023-08-06T19:58:05.124Z", |
"completedTime": false, |
"timeTaken": 41530161, |
"details": { |
"travel-sample": { |
"vbucketLevelInfo": { |
"move": { |
"averageTime": 0, |
"totalCount": 654, |
"remainingCount": 654 |
},
|
"backfill": { |
"averageTime": 0 |
},
|
"takeover": { |
"averageTime": 0 |
},
|
"persistence": { |
"averageTime": 0 |
}
|
},
|
"replicationInfo": { |
"ns_1@svc-dqisea-node-001.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com": { |
"inDocsTotal": 39256, |
"inDocsLeft": 39256, |
"outDocsTotal": 1979493, |
"outDocsLeft": 1979493 |
},
|
"ns_1@svc-dqisea-node-002.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com": { |
"inDocsTotal": 46045, |
"inDocsLeft": 46045, |
"outDocsTotal": 2075306, |
"outDocsLeft": 2075306 |
},
|
"ns_1@svc-dqisea-node-004.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com": { |
"inDocsTotal": 3150359, |
"inDocsLeft": 3150359, |
"outDocsTotal": 0, |
"outDocsLeft": 0 |
},
|
"ns_1@svc-dqisea-node-005.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com": { |
"inDocsTotal": 3168517, |
"inDocsLeft": 3168517, |
"outDocsTotal": 0, |
"outDocsLeft": 0 |
},
|
"ns_1@svc-dqisea-node-003.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com": { |
"inDocsTotal": 405, |
"inDocsLeft": 405, |
"outDocsTotal": 2349783, |
"outDocsLeft": 2349783 |
}
|
},
|
"startTime": "2023-08-06T19:58:06.771Z", |
"completedTime": false, |
"timeTaken": 41528514 |
}
|
}
|
},
|
"query": { |
"startTime": false, |
"completedTime": false, |
"timeTaken": false |
}
|
},
|
"rebalanceId": "b8ed5c2648b6498f870e5150bea66185", |
"nodesInfo": { |
"active_nodes": [ |
"ns_1@svc-dqisea-node-001.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com", |
"ns_1@svc-dqisea-node-002.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com", |
"ns_1@svc-dqisea-node-003.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com", |
"ns_1@svc-dqisea-node-004.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com", |
"ns_1@svc-dqisea-node-005.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com" |
],
|
"keep_nodes": [ |
"ns_1@svc-dqisea-node-001.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com", |
"ns_1@svc-dqisea-node-002.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com", |
"ns_1@svc-dqisea-node-003.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com", |
"ns_1@svc-dqisea-node-004.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com", |
"ns_1@svc-dqisea-node-005.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com" |
],
|
"eject_nodes": [], |
"delta_nodes": [], |
"failed_nodes": [] |
},
|
"masterNode": "ns_1@svc-dqisea-node-003.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com" |
},
|
{
|
"node": "ns_1@svc-dqisea-node-003.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com", |
"type": "clusterLogsCollection", |
"perNode": { |
"ns_1@svc-dqisea-node-001.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com": { |
"path": "/opt/couchbase/var/lib/couchbase/tmp/collectinfo-2023-08-07T055301-ns_1@svc-dqisea-node-001.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com-redacted.zip", |
"status": "uploaded", |
"url": "https://cb-engineering.s3.amazonaws.com/aman/collectinfo-2023-08-07T055301-ns_1%40svc-dqisea-node-001.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com-redacted.zip" |
},
|
"ns_1@svc-dqisea-node-002.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com": { |
"path": "/opt/couchbase/var/lib/couchbase/tmp/collectinfo-2023-08-07T055301-ns_1@svc-dqisea-node-002.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com-redacted.zip", |
"status": "uploaded", |
"url": "https://cb-engineering.s3.amazonaws.com/aman/collectinfo-2023-08-07T055301-ns_1%40svc-dqisea-node-002.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com-redacted.zip" |
},
|
"ns_1@svc-dqisea-node-003.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com": { |
"path": "/opt/couchbase/var/lib/couchbase/tmp/collectinfo-2023-08-07T055301-ns_1@svc-dqisea-node-003.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com-redacted.zip", |
"status": "uploaded", |
"url": "https://cb-engineering.s3.amazonaws.com/aman/collectinfo-2023-08-07T055301-ns_1%40svc-dqisea-node-003.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com-redacted.zip" |
},
|
"ns_1@svc-dqisea-node-004.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com": { |
"path": "/opt/couchbase/var/lib/couchbase/tmp/collectinfo-2023-08-07T055301-ns_1@svc-dqisea-node-004.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com-redacted.zip", |
"status": "uploaded", |
"url": "https://cb-engineering.s3.amazonaws.com/aman/collectinfo-2023-08-07T055301-ns_1%40svc-dqisea-node-004.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com-redacted.zip" |
},
|
"ns_1@svc-dqisea-node-005.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com": { |
"path": "/opt/couchbase/var/lib/couchbase/tmp/collectinfo-2023-08-07T055301-ns_1@svc-dqisea-node-005.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com-redacted.zip", |
"status": "uploaded", |
"url": "https://cb-engineering.s3.amazonaws.com/aman/collectinfo-2023-08-07T055301-ns_1%40svc-dqisea-node-005.kzyut3zkiogzbaup.sandbox.nonprod-project-avengers.com-redacted.zip" |
}
|
},
|
"progress": 100, |
"ts": "2023-08-07 05:53:01", |
"status": "completed" |
}
|
]
|
Attachments
Issue Links
- relates to
-
MB-57735 [Import] Add an --eviction-policy flag
-
- Closed
-