Details
-
Bug
-
Resolution: Won't Fix
-
Major
-
7.1.0
-
Triaged
-
1
-
No
-
Tools 2021 Dec
Description
7.1.0-1861
Test:
-test tests/integration/neo/test_neo_couchstore_milestone3.yml -scope tests/integration/neo/scope_couchstore.yml
Scale 3
Iteration 2
Backup to s3 task failed with error:
{
|
"task_name": "backup-1",
|
"status": "failed",
|
"start": "2021-12-09T22:00:45.374285939-08:00",
|
"end": "2021-12-09T22:42:46.071862553-08:00",
|
"node_runs": [
|
{
|
"node_id": "a9f4f45f4650e1e32e7c46450c91dfcf",
|
"status": "failed",
|
"start": "2021-12-09T22:00:45.461618317-08:00",
|
"end": "2021-12-09T22:42:45.996229751-08:00",
|
"error": "exit status 1",
|
"progress": 81.86601192564012,
|
"stats": {
|
"id": "880f5fc6-c058-4850-8935-8589d0e7e578",
|
"current_transfer": 1,
|
"total_transfers": 1,
|
"transfers": [
|
{
|
"description": "Backing up to 2021-12-09T22_02_37.721198072-08_00",
|
"stats": {
|
"started_at": 1639116156529006800,
|
"buckets": {
|
"N1QL_SYSTEM_BUCKET": {
|
"estimated_total_items": 20924,
|
"total_items": 551,
|
"total_vbuckets": 1024,
|
"vbuckets_complete": 1024,
|
"bytes_received": 508649,
|
"snapshot_markers_received": 1024,
|
"failover_logs_received": 1024,
|
"mutations_received": 280,
|
"deletions_received": 271,
|
"started_at": 1639117361287296500,
|
"finished_at": 1639117376370721000,
|
"complete": true
|
},
|
"bucket1": {
|
"estimated_total_items": 142301443,
|
"total_items": 5705041,
|
"total_vbuckets": 1024,
|
"vbuckets_complete": 1024,
|
"bytes_received": 4028638419,
|
"snapshot_markers_received": 1024,
|
"failover_logs_received": 1024,
|
"mutations_received": 3304813,
|
"deletions_received": 2405135,
|
"started_at": 1639117578014752300,
|
"finished_at": 1639117683687727600,
|
"complete": true
|
},
|
"bucket2": {
|
"estimated_total_items": 189170231,
|
"total_items": 3255353,
|
"total_vbuckets": 1024,
|
"vbuckets_complete": 1024,
|
"bytes_received": 248140800,
|
"snapshot_markers_received": 1024,
|
"failover_logs_received": 1024,
|
"mutations_received": 1064143,
|
"deletions_received": 2192143,
|
"started_at": 1639117888070114000,
|
"finished_at": 1639117906013347000,
|
"complete": true
|
},
|
"bucket3": {
|
"estimated_total_items": 175607523,
|
"total_items": 2669538,
|
"total_vbuckets": 1024,
|
"vbuckets_complete": 1024,
|
"bytes_received": 257163354,
|
"snapshot_markers_received": 1024,
|
"failover_logs_received": 1024,
|
"mutations_received": 44402,
|
"deletions_received": 2627527,
|
"started_at": 1639118104779701800,
|
"finished_at": 1639118123047515100,
|
"complete": true
|
},
|
"bucket4": {
|
"estimated_total_items": 7662647,
|
"total_items": 1698645,
|
"total_vbuckets": 1024,
|
"vbuckets_complete": 1024,
|
"bytes_received": 959229212,
|
"snapshot_markers_received": 1024,
|
"failover_logs_received": 1024,
|
"mutations_received": 1187703,
|
"deletions_received": 510942,
|
"started_at": 1639118322050438700,
|
"finished_at": 1639118353405795300,
|
"complete": true
|
},
|
"bucket5": {
|
"estimated_total_items": 57734107,
|
"total_items": 1838703,
|
"total_vbuckets": 1024,
|
"vbuckets_complete": 1024,
|
"bytes_received": 1396788302,
|
"snapshot_markers_received": 1024,
|
"failover_logs_received": 1024,
|
"mutations_received": 1774320,
|
"deletions_received": 64383,
|
"started_at": 1639116163134487300,
|
"finished_at": 1639116222250075400,
|
"complete": true
|
},
|
"bucket6": {
|
"estimated_total_items": 27698381,
|
"total_items": 1838703,
|
"total_vbuckets": 1024,
|
"vbuckets_complete": 1024,
|
"bytes_received": 1324628452,
|
"snapshot_markers_received": 1024,
|
"failover_logs_received": 1024,
|
"mutations_received": 1675953,
|
"deletions_received": 162750,
|
"started_at": 1639116421488955600,
|
"finished_at": 1639116487784147700,
|
"complete": true
|
},
|
"bucket7": {
|
"estimated_total_items": 80054400,
|
"total_items": 88342,
|
"total_vbuckets": 1024,
|
"vbuckets_complete": 1024,
|
"bytes_received": 8417540,
|
"snapshot_markers_received": 1024,
|
"failover_logs_received": 1024,
|
"mutations_received": 42961,
|
"deletions_received": 45386,
|
"started_at": 1639116692797386000,
|
"finished_at": 1639116706217546800,
|
"complete": true
|
},
|
"default": {
|
"estimated_total_items": 839355116,
|
"total_items": 11595361,
|
"total_vbuckets": 1024,
|
"vbuckets_complete": 1024,
|
"bytes_received": 13135962321,
|
"snapshot_markers_received": 1024,
|
"failover_logs_received": 1024,
|
"mutations_received": 3001143,
|
"deletions_received": 8610681,
|
"started_at": 1639116915015074600,
|
"finished_at": 1639117163217422300,
|
"complete": true
|
}
|
}
|
},
|
"progress": 81.86601192564012,
|
"eta": "2021-12-09T22:51:35.941415123-08:00"
|
}
|
],
|
"progress": 81.86601192564012,
|
"eta": "2021-12-09T22:51:35.941415123-08:00"
|
},
|
"error_code": 2
|
}
|
],
|
"error": "exit status 1",
|
"error_code": 2,
|
"type": "BACKUP",
|
"show": true
|
}
|
From CBM logs:
2021-12-09T22:42:31.345-08:00 (Plan) (Data) Deciding which key value data to transfer for bucket 'bucket8'
|
2021-12-09T22:42:31.345-08:00 (Couchbase) Creating memcached connection to cluster 'couchbase://172.23.104.67:11210,172.23.104.69:11210,172.23.106.100:11210,172.23.108.103:11210,172.23.96.251:11210,172.23.97.119:11210,172.23.97.121:11210,172.23.97.122:11210,172.23.99.20:11210,172.23.99.25:11210'
|
2021-12-09T22:42:31.413-08:00 (Couchbase) Successfully created memcached connection to cluster 'couchbase://172.23.104.67:11210,172.23.104.69:11210,172.23.106.100:11210,172.23.108.103:11210,172.23.96.251:11210,172.23.97.119:11210,172.23.97.121:11210,172.23.97.122:11210,172.23.99.20:11210,172.23.99.25:11210'
|
2021-12-09T22:42:31.723-08:00 WARN: (Couchbase) Unexpected error 'failed to fetch the required sequence numbers for all available vBuckets, please try again later' while trying to get sequence numbers, will retry -- couchbase.GetAllBucketSeqNosWithRetries.func2() at sequence_numbers.go:66
|
2021-12-09T22:42:32.020-08:00 WARN: (Couchbase) Unexpected error 'failed to fetch the required sequence numbers for all available vBuckets, please try again later' while trying to get sequence numbers, will retry -- couchbase.GetAllBucketSeqNosWithRetries.func2() at sequence_numbers.go:66
|
2021-12-09T22:42:32.369-08:00 WARN: (Couchbase) Unexpected error 'failed to fetch the required sequence numbers for all available vBuckets, please try again later' while trying to get sequence numbers, will retry -- couchbase.GetAllBucketSeqNosWithRetries.func2() at sequence_numbers.go:66
|
2021-12-09T22:42:32.777-08:00 WARN: (Couchbase) Unexpected error 'failed to fetch the required sequence numbers for all available vBuckets, please try again later' while trying to get sequence numbers, will retry -- couchbase.GetAllBucketSeqNosWithRetries.func2() at sequence_numbers.go:66
|
2021-12-09T22:42:33.294-08:00 (Cmd) Error backing up cluster: failed to execute cluster operations: failed to execute bucket operation for bucket 'bucket8': failed to transfer bucket data for bucket 'bucket8': failed to transfer data ranges: failed to get source data ranges: exhausted retry count after 5 attempts: failed to fetch the required sequence numbers for all available vBuckets, please try again later
|
2021-12-10T04:05:16.407-08:00 (Cmd) cbbackupmgr version 7.1.0-1861 Hostname: localhost.localdomain OS: linux Version: 3.10.0-1062.18.1.el7.x86_64 Arch: amd64 vCPU: 8 Memory: 23564947456 (21.95GiB)
|
2021-12-10T04:05:16.409-08:00 (Cmd) backup --cluster http://172.23.123.28:8091 --archive s3://backup-service-testing/longe
|
Cluster config:
########## Cluster config ##################
|
###### fts : 2 ===== > [172.23.104.155:8091 172.23.96.148:8091] ###########
|
###### eventing : 2 ===== > [172.23.104.5:8091 172.23.98.135:8091] ###########
|
###### kv : 11 ===== > [172.23.104.67:8091 172.23.104.69:8091 172.23.108.103:8091 172.23.96.251:8091 172.23.97.119:8091 172.23.97.121:8091 172.23.97.122:8091 172.23.97.239:8091 172.23.99.20:8091 172.23.99.21:8091 172.23.99.25:8091] ###########
|
###### index : 6 ===== > [172.23.104.70:8091 172.23.120.107:8091 172.23.120.98:8091 172.23.121.117:8091 172.23.96.252:8091 172.23.96.253:8091] ###########
|
###### cbas : 1 ===== > [172.23.106.188:8091] ###########
|
###### backup : 1 ===== > [172.23.123.28:8091] ###########
|
###### n1ql : 2 ===== > [172.23.104.137:8091 172.23.99.11:8091] ###########
|
There were two rebalances happening during this time:
[2021-12-09T21:57:02-08:00, sequoiatools/couchbase-cli:7.1:c5008d] rebalance -c 172.23.108.103:8091 -u Administrator -p password
|
[2021-12-09T22:19:07-08:00, sequoiatools/cmd:6b22ee] 60
|
[2021-12-09T22:20:14-08:00, sequoiatools/cmd:e24eeb] 600
|
[2021-12-09T22:30:20-08:00, appropriate/curl:eb1b0c] -u Administrator:password -X POST http://172.23.108.103:8091/settings/replications/40b911a35e0c1bda7bb15bd3944afdef/bucket8/bucket8 -d pauseRequested=true
|
[2021-12-09T22:30:26-08:00, sequoiatools/cmd:12b96f] 300
|
[2021-12-09T22:35:32-08:00, appropriate/curl:ff3450] -u Administrator:password -X POST http://172.23.108.103:8091/settings/replications/40b911a35e0c1bda7bb15bd3944afdef/bucket8/bucket8 -d pauseRequested=false
|
[2021-12-09T22:35:39-08:00, sequoiatools/gideon:922cfb] kv --ops 1500 --create 100 --expire 100 --ttl 660 --hosts 172.23.108.103 --bucket default --sizes 64
|
[2021-12-09T22:35:44-08:00, sequoiatools/pillowfight:7.0:39846d] -U couchbase://172.23.108.103/default?select_bucket=true -I 3000 -B 300 -t 4 -c 100 -P password
|
[2021-12-09T22:36:41-08:00, sequoiatools/couchbase-cli:7.1:45e395] server-add -c 172.23.108.103:8091 --server-add https://172.23.106.100 -u Administrator -p password --server-add-username Administrator --server-add-password password --services data
|
[2021-12-09T22:38:08-08:00, sequoiatools/couchbase-cli:7.1:fc2dd2] failover -c 172.23.108.103:8091 --server-failover 172.23.97.239:8091 -u Administrator -p password
|
[2021-12-09T22:40:56-08:00, sequoiatools/couchbase-cli:7.1:085dbe] failover -c 172.23.108.103:8091 --server-failover 172.23.99.21:8091 -u Administrator -p password --hard
|
[2021-12-09T22:41:07-08:00, sequoiatools/couchbase-cli:7.1:4ae20c] rebalance -c 172.23.108.103:8091 -u Administrator -p password
|
[2021-12-09T23:10:26-08:00, sequoiatools/cmd:7e3778] 60
|
Logs:
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1639119774/collectinfo-2021-12-10T070257-ns_1%40172.23.104.137.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1639119774/collectinfo-2021-12-10T070257-ns_1%40172.23.104.155.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1639119774/collectinfo-2021-12-10T070257-ns_1%40172.23.104.5.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1639119774/collectinfo-2021-12-10T070257-ns_1%40172.23.104.67.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1639119774/collectinfo-2021-12-10T070257-ns_1%40172.23.104.69.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1639119774/collectinfo-2021-12-10T070257-ns_1%40172.23.104.70.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1639119774/collectinfo-2021-12-10T070257-ns_1%40172.23.106.100.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1639119774/collectinfo-2021-12-10T070257-ns_1%40172.23.106.188.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1639119774/collectinfo-2021-12-10T070257-ns_1%40172.23.108.103.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1639119774/collectinfo-2021-12-10T070257-ns_1%40172.23.120.107.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1639119774/collectinfo-2021-12-10T070257-ns_1%40172.23.120.98.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1639119774/collectinfo-2021-12-10T070257-ns_1%40172.23.121.117.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1639119774/collectinfo-2021-12-10T070257-ns_1%40172.23.123.28.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1639119774/collectinfo-2021-12-10T070257-ns_1%40172.23.96.148.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1639119774/collectinfo-2021-12-10T070257-ns_1%40172.23.96.251.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1639119774/collectinfo-2021-12-10T070257-ns_1%40172.23.96.252.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1639119774/collectinfo-2021-12-10T070257-ns_1%40172.23.96.253.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1639119774/collectinfo-2021-12-10T070257-ns_1%40172.23.97.119.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1639119774/collectinfo-2021-12-10T070257-ns_1%40172.23.97.121.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1639119774/collectinfo-2021-12-10T070257-ns_1%40172.23.97.122.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1639119774/collectinfo-2021-12-10T070257-ns_1%40172.23.97.242.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1639119774/collectinfo-2021-12-10T070257-ns_1%40172.23.98.135.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1639119774/collectinfo-2021-12-10T070257-ns_1%40172.23.99.11.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1639119774/collectinfo-2021-12-10T070257-ns_1%40172.23.99.20.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1639119774/collectinfo-2021-12-10T070257-ns_1%40172.23.99.25.zip
Attachments
Issue Links
- relates to
-
-
- Closed
-