Details
-
Bug
-
Resolution: Fixed
-
Major
-
7.0.0
-
Untriaged
-
1
-
Unknown
Description
7.0.0-5295
Test:
-test tests/integration/cheshirecat/test_cheshirecat_kv_gsi_coll_xdcr_backup_sgw_fts_itemct_txns_eventing_cbas_scale3.yml -scope tests/integration/cheshirecat/scope_cheshirecat_with_backup.yml
12th day 7th iteration
backup task failure:
{
|
"task_name": "backup-1",
|
"status": "failed",
|
"start": "2021-06-24T04:00:52.582699858-07:00",
|
"end": "2021-06-24T04:06:14.357283886-07:00",
|
"node_runs": [
|
{
|
"node_id": "301d9c46c75f781a445f5ccc86627f95",
|
"status": "failed",
|
"start": "2021-06-24T04:00:52.680763742-07:00",
|
"end": "2021-06-24T04:06:14.272473317-07:00",
|
"error": "exit status 1: failed to execute cluster operations: failed to execute bucket operation for bucket 'bucket9': failed to transfer bucket data for bucket 'bucket9': failed to transfer key value data: failed to update source scope/collection mappings: failed to get source collection manifest: failed to wait for manifest id 7082: failed to execute request: http client failed to dispatch/receive request/response: Post \"http://172.23.104.137:8091/pools/default/buckets/bucket9/scopes/%40ensureManifest/1baa\": context deadline exceeded",
|
"progress": 0,
|
"stats": {
|
"error": "failed to execute cluster operations: failed to execute bucket operation for bucket 'bucket9': failed to transfer bucket data for bucket 'bucket9': failed to transfer key value data: failed to update source scope/collection mappings: failed to get source collection manifest: failed to wait for manifest id 7082: failed to execute request: http client failed to dispatch/receive request/response: Post \"http://172.23.104.137:8091/pools/default/buckets/bucket9/scopes/%40ensureManifest/1baa\": context deadline exceeded",
|
"stats": {
|
"started_at": 1624532453005698000,
|
"buckets": {
|
"ITEM": {
|
"estimated_total_items": 1256546025,
|
"total_items": 6236770,
|
"total_vbuckets": 1024,
|
"vbuckets_complete": 1024,
|
"bytes_received": 598892232,
|
"snapshot_markers_received": 1024,
|
"snapshot_ends_received": 1024,
|
"failover_logs_received": 1024,
|
"mutations_received": 58837,
|
"deletions_received": 6178993,
|
"started_at": 1624532579841792800,
|
"finished_at": 1624532608806792700,
|
"complete": true,
|
"errored": false
|
},
|
"WAREHOUSE": {
|
"estimated_total_items": 1115623764,
|
"total_items": 9464205,
|
"total_vbuckets": 1024,
|
"vbuckets_complete": 1024,
|
"bytes_received": 4573243797,
|
"snapshot_markers_received": 1024,
|
"snapshot_ends_received": 1024,
|
"failover_logs_received": 1024,
|
"mutations_received": 3352717,
|
"deletions_received": 6140212,
|
"started_at": 1624532612596392200,
|
"finished_at": 1624532661774220800,
|
"complete": true,
|
"errored": false
|
},
|
"bucket5": {
|
"estimated_total_items": 261590869,
|
"total_items": 1619351,
|
"total_vbuckets": 1024,
|
"vbuckets_complete": 1024,
|
"bytes_received": 874631011,
|
"snapshot_markers_received": 1024,
|
"snapshot_ends_received": 1024,
|
"failover_logs_received": 1024,
|
"mutations_received": 1078459,
|
"deletions_received": 540892,
|
"started_at": 1624532667058550300,
|
"finished_at": 1624532688710052000,
|
"complete": true,
|
"errored": false
|
},
|
"bucket6": {
|
"estimated_total_items": 178110187,
|
"total_items": 1653282,
|
"total_vbuckets": 1024,
|
"vbuckets_complete": 1024,
|
"bytes_received": 882300612,
|
"snapshot_markers_received": 1024,
|
"snapshot_ends_received": 1024,
|
"failover_logs_received": 1024,
|
"mutations_received": 1086240,
|
"deletions_received": 567042,
|
"started_at": 1624532692715339300,
|
"finished_at": 1624532708478172200,
|
"complete": true,
|
"errored": false
|
},
|
"bucket7": {
|
"estimated_total_items": 775626571,
|
"total_items": 239588,
|
"total_vbuckets": 1024,
|
"vbuckets_complete": 1024,
|
"bytes_received": 18500090,
|
"snapshot_markers_received": 1024,
|
"snapshot_ends_received": 1024,
|
"failover_logs_received": 1024,
|
"mutations_received": 13,
|
"deletions_received": 239575,
|
"started_at": 1624532464043596800,
|
"finished_at": 1624532468207193900,
|
"complete": true,
|
"errored": false
|
},
|
"bucket8": {
|
"estimated_total_items": 7993614,
|
"total_items": 34164,
|
"total_vbuckets": 1024,
|
"vbuckets_complete": 1024,
|
"bytes_received": 131984,
|
"snapshot_markers_received": 1024,
|
"snapshot_ends_received": 1024,
|
"failover_logs_received": 1024,
|
"started_at": 1624532471975479000,
|
"finished_at": 1624532474707847000,
|
"complete": true,
|
"errored": false
|
},
|
"default": {
|
"estimated_total_items": 4234421084,
|
"total_items": 13457224,
|
"total_vbuckets": 1024,
|
"vbuckets_complete": 1024,
|
"bytes_received": 12777513503,
|
"snapshot_markers_received": 1024,
|
"snapshot_ends_received": 1024,
|
"failover_logs_received": 1024,
|
"mutations_received": 2235205,
|
"deletions_received": 11240675,
|
"started_at": 1624532478862219800,
|
"finished_at": 1624532576001258200,
|
"complete": true,
|
"errored": false
|
}
|
},
|
"complete": false
|
}
|
},
|
"error_code": 2
|
}
|
],
|
"error": "exit status 1: failed to execute cluster operations: failed to execute bucket operation for bucket 'bucket9': failed to transfer bucket data for bucket 'bucket9': failed to transfer key value data: failed to update source scope/collection mappings: failed to get source collection manifest: failed to wait for manifest id 7082: failed to execute request: http client failed to dispatch/receive request/response: Post \"http://172.23.104.137:8091/pools/default/buckets/bucket9/scopes/%40ensureManifest/1baa\": context deadline exceeded",
|
"error_code": 2,
|
"type": "BACKUP",
|
"show": true
|
}
|
Subsequent backups succeeded - during this time, the test was doing a rebalance:
[2021-06-24T03:05:46-07:00, sequoiatools/couchbase-cli:7.0:c4f8d4] rebalance -c 172.23.108.103:8091 -u Administrator -p password
[2021-06-24T08:05:17-07:00, sequoiatools/cmd:0b0a5e] 60
Logs:
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1624539450/collectinfo-2021-06-24T125733-ns_1%40172.23.104.137.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1624539450/collectinfo-2021-06-24T125733-ns_1%40172.23.104.155.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1624539450/collectinfo-2021-06-24T125733-ns_1%40172.23.104.157.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1624539450/collectinfo-2021-06-24T125733-ns_1%40172.23.104.5.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1624539450/collectinfo-2021-06-24T125733-ns_1%40172.23.104.69.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1624539450/collectinfo-2021-06-24T125733-ns_1%40172.23.104.70.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1624539450/collectinfo-2021-06-24T125733-ns_1%40172.23.105.107.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1624539450/collectinfo-2021-06-24T125733-ns_1%40172.23.105.111.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1624539450/collectinfo-2021-06-24T125733-ns_1%40172.23.106.188.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1624539450/collectinfo-2021-06-24T125733-ns_1%40172.23.108.103.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1624539450/collectinfo-2021-06-24T125733-ns_1%40172.23.120.245.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1624539450/collectinfo-2021-06-24T125733-ns_1%40172.23.121.117.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1624539450/collectinfo-2021-06-24T125733-ns_1%40172.23.121.3.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1624539450/collectinfo-2021-06-24T125733-ns_1%40172.23.123.27.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1624539450/collectinfo-2021-06-24T125733-ns_1%40172.23.123.28.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1624539450/collectinfo-2021-06-24T125733-ns_1%40172.23.96.148.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1624539450/collectinfo-2021-06-24T125733-ns_1%40172.23.96.251.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1624539450/collectinfo-2021-06-24T125733-ns_1%40172.23.96.252.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1624539450/collectinfo-2021-06-24T125733-ns_1%40172.23.96.253.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1624539450/collectinfo-2021-06-24T125733-ns_1%40172.23.97.119.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1624539450/collectinfo-2021-06-24T125733-ns_1%40172.23.97.122.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1624539450/collectinfo-2021-06-24T125733-ns_1%40172.23.97.239.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1624539450/collectinfo-2021-06-24T125733-ns_1%40172.23.98.135.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1624539450/collectinfo-2021-06-24T125733-ns_1%40172.23.99.11.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1624539450/collectinfo-2021-06-24T125733-ns_1%40172.23.99.20.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1624539450/collectinfo-2021-06-24T125733-ns_1%40172.23.99.21.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1624539450/collectinfo-2021-06-24T125733-ns_1%40172.23.99.25.zip