Details
-
Bug
-
Resolution: Not a Bug
-
Critical
-
7.1.0
-
Untriaged
-
1
-
Yes
Description
7.1.0-2534 (Rc3)
Test:
-test tests/integration/neo/test_neo_couchstore_milestone4.yml -scope tests/integration/neo/scope_couchstore.yml
Following issue observed with local backup:
{
|
"task_name": "backup-1", |
"status": "failed", |
"start": "2022-03-31T01:00:41.471941428-07:00", |
"end": "2022-03-31T01:02:45.225100363-07:00", |
"node_runs": [ |
{
|
"node_id": "bfa1066b243e98ef63aa76c81b9ce123", |
"status": "failed", |
"start": "2022-03-31T01:00:41.736847177-07:00", |
"end": "2022-03-31T01:02:44.98071655-07:00", |
"error": "exit status 1: failed to execute cluster operations: failed to execute bucket operation for bucket 'bucket7': failed to transfer Analytics for bucket 'bucket7': failed to get Analytics metadata: failed to get analytics metadata: failed to execute request: failed to execute request: exhausted retry count after 3 retries, last error: internal server error executing 'GET' request to '/api/v1/bucket/bucket7/backup': {\n\t\"version\": 2,\n\t\"requestID\": \"6596b60a-45f1-473b-9b95-527377cc7a76\",\n\t\"errors\": [{ \n\t\t\"code\": 23000,\t\t\"msg\": \"Analytics Service is temporarily unavailable\"\t} \n\t],\n\t\"status\": \"fatal\",\n\t\"metrics\": {\n\t\t\"elapsedTime\": \"8.737995ms\",\n\t\t\"executionTime\": \"2.332567ms\",\n\t\t\"resultCount\": 0,\n\t\t\"resultSize\": 0,\n\t\t\"processedObjects\": 0,\n\t\t\"errorCount\": 1\n\t}\n}\n", |
"progress": 0, |
"stats": { |
"error": "failed to execute cluster operations: failed to execute bucket operation for bucket 'bucket7': failed to transfer Analytics for bucket 'bucket7': failed to get Analytics metadata: failed to get analytics metadata: failed to execute request: failed to execute request: exhausted retry count after 3 retries, last error: internal server error executing 'GET' request to '/api/v1/bucket/bucket7/backup': {\n\t\"version\": 2,\n\t\"requestID\": \"6596b60a-45f1-473b-9b95-527377cc7a76\",\n\t\"errors\": [{ \n\t\t\"code\": 23000,\t\t\"msg\": \"Analytics Service is temporarily unavailable\"\t} \n\t],\n\t\"status\": \"fatal\",\n\t\"metrics\": {\n\t\t\"elapsedTime\": \"8.737995ms\",\n\t\t\"executionTime\": \"2.332567ms\",\n\t\t\"resultCount\": 0,\n\t\t\"resultSize\": 0,\n\t\t\"processedObjects\": 0,\n\t\t\"errorCount\": 1\n\t}\n}\n", |
"stats": { |
"started_at": 1648713642861233200, |
"buckets": { |
"N1QL_SYSTEM_BUCKET": { |
"estimated_total_items": 13601, |
"total_items": 681, |
"total_vbuckets": 1024, |
"vbuckets_complete": 1024, |
"bytes_received": 508298, |
"snapshot_markers_received": 1024, |
"failover_logs_received": 1024, |
"mutations_received": 219, |
"deletions_received": 462, |
"started_at": 1648713660297268500, |
"finished_at": 1648713663897422300, |
"complete": true |
},
|
"bucket1": { |
"estimated_total_items": 303701253, |
"total_items": 4497330, |
"total_vbuckets": 1024, |
"vbuckets_complete": 1024, |
"bytes_received": 3504944425, |
"snapshot_markers_received": 1024, |
"failover_logs_received": 1024, |
"mutations_received": 2881526, |
"deletions_received": 1619117, |
"started_at": 1648713668390584000, |
"finished_at": 1648713685151503000, |
"complete": true |
},
|
"bucket2": { |
"estimated_total_items": 452241217, |
"total_items": 6949797, |
"total_vbuckets": 1024, |
"vbuckets_complete": 1024, |
"bytes_received": 468143838, |
"snapshot_markers_received": 1024, |
"failover_logs_received": 1024, |
"mutations_received": 949825, |
"deletions_received": 6001249, |
"started_at": 1648713737869911300, |
"finished_at": 1648713752476223000, |
"complete": true |
},
|
"bucket3": { |
"estimated_total_items": 479400782, |
"total_items": 3290309, |
"total_vbuckets": 1024, |
"vbuckets_complete": 1024, |
"bytes_received": 311496061, |
"snapshot_markers_received": 1024, |
"failover_logs_received": 1024, |
"mutations_received": 29980, |
"deletions_received": 3244226, |
"started_at": 1648713688395884300, |
"finished_at": 1648713698094012700, |
"complete": true |
},
|
"bucket4": { |
"estimated_total_items": 27300963, |
"total_items": 1817654, |
"total_vbuckets": 1024, |
"vbuckets_complete": 1024, |
"bytes_received": 1428666934, |
"snapshot_markers_received": 1024, |
"failover_logs_received": 1024, |
"mutations_received": 1817654, |
"started_at": 1648713703646307600, |
"finished_at": 1648713712436376600, |
"complete": true |
},
|
"bucket5": { |
"estimated_total_items": 41910982, |
"total_items": 1818181, |
"total_vbuckets": 1024, |
"vbuckets_complete": 1024, |
"bytes_received": 1309530609, |
"snapshot_markers_received": 1024, |
"failover_logs_received": 1024, |
"mutations_received": 1655431, |
"deletions_received": 162750, |
"started_at": 1648713756765534200, |
"finished_at": 1648713763967859200, |
"complete": true |
},
|
"bucket6": { |
"estimated_total_items": 50324648, |
"total_items": 1818181, |
"total_vbuckets": 1024, |
"vbuckets_complete": 1024, |
"bytes_received": 1309961495, |
"snapshot_markers_received": 1024, |
"failover_logs_received": 1024, |
"mutations_received": 1655431, |
"deletions_received": 162750, |
"started_at": 1648713717641000000, |
"finished_at": 1648713725528399000, |
"complete": true |
},
|
"bucket8": { |
"estimated_total_items": 1738832, |
"total_items": 44859, |
"total_vbuckets": 1024, |
"vbuckets_complete": 1024, |
"bytes_received": 80944, |
"snapshot_markers_received": 1024, |
"failover_logs_received": 1024, |
"started_at": 1648713730707312000, |
"finished_at": 1648713733437451500, |
"complete": true |
},
|
"bucket9": { |
"estimated_total_items": 1700460, |
"total_items": 18476, |
"total_vbuckets": 1024, |
"vbuckets_complete": 1024, |
"bytes_received": 80944, |
"snapshot_markers_received": 1024, |
"failover_logs_received": 1024, |
"started_at": 1648713652472698000, |
"finished_at": 1648713655555433000, |
"complete": true |
}
|
}
|
}
|
},
|
"error_code": 2 |
}
|
],
|
"error": "exit status 1: failed to execute cluster operations: failed to execute bucket operation for bucket 'bucket7': failed to transfer Analytics for bucket 'bucket7': failed to get Analytics metadata: failed to get analytics metadata: failed to execute request: failed to execute request: exhausted retry count after 3 retries, last error: internal server error executing 'GET' request to '/api/v1/bucket/bucket7/backup': {\n\t\"version\": 2,\n\t\"requestID\": \"6596b60a-45f1-473b-9b95-527377cc7a76\",\n\t\"errors\": [{ \n\t\t\"code\": 23000,\t\t\"msg\": \"Analytics Service is temporarily unavailable\"\t} \n\t],\n\t\"status\": \"fatal\",\n\t\"metrics\": {\n\t\t\"elapsedTime\": \"8.737995ms\",\n\t\t\"executionTime\": \"2.332567ms\",\n\t\t\"resultCount\": 0,\n\t\t\"resultSize\": 0,\n\t\t\"processedObjects\": 0,\n\t\t\"errorCount\": 1\n\t}\n}\n", |
"error_code": 2, |
"type": "BACKUP", |
"show": true |
}
|
During this time, test was removing an analytics node and rebalance was going on:
[2022-03-31T01:00:36-07:00, sequoiatools/couchbase-cli:7.1:f11ef5] rebalance -c 172.23.108.103:8091 --server-remove 172.23.106.188 -u Administrator -p password [2022-03-31T01:03:14-07:00, sequoiatools/cmd:082258] 60
Cluster config:
backup : 1 ===== > [172.23.104.69:8091] ########### ###### kv : 11 ===== > [172.23.105.107:8091 172.23.106.100:8091 172.23.108.103:8091 172.23.121.117:8091 172.23.96.253:8091 172.23.97.119:8091 172.23.97.121:8091 172.23.97.122:8091 172.23.99.20:8091 172.23.99.21:8091 172.23.99.25:8091] ########### ###### fts : 2 ===== > [172.23.105.111:8091 172.23.96.148:8091] ########### ###### cbas : 3 ===== > [172.23.105.168:8091 172.23.106.188:8091 172.23.97.239:8091] ########### ###### n1ql : 2 ===== > [172.23.104.137:8091 172.23.99.11:8091] ########### ###### index : 6 ===== > [172.23.104.155:8091 172.23.120.107:8091 172.23.120.245:8091 172.23.123.28:8091 172.23.96.251:8091 172.23.96.252:8091] ########### ###### eventing : 3 ===== > [172.23.104.5:8091 172.23.104.67:8091 172.23.96.192:8091] ########### |
Server logs:
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1648714993/collectinfo-2022-03-31T082317-ns_1%40172.23.104.137.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1648714993/collectinfo-2022-03-31T082317-ns_1%40172.23.104.155.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1648714993/collectinfo-2022-03-31T082317-ns_1%40172.23.104.5.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1648714993/collectinfo-2022-03-31T082317-ns_1%40172.23.104.67.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1648714993/collectinfo-2022-03-31T082317-ns_1%40172.23.104.69.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1648714993/collectinfo-2022-03-31T082317-ns_1%40172.23.104.70.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1648714993/collectinfo-2022-03-31T082317-ns_1%40172.23.105.107.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1648714993/collectinfo-2022-03-31T082317-ns_1%40172.23.105.111.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1648714993/collectinfo-2022-03-31T082317-ns_1%40172.23.105.168.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1648714993/collectinfo-2022-03-31T082317-ns_1%40172.23.106.100.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1648714993/collectinfo-2022-03-31T082317-ns_1%40172.23.106.188.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1648714993/collectinfo-2022-03-31T082317-ns_1%40172.23.108.103.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1648714993/collectinfo-2022-03-31T082317-ns_1%40172.23.120.107.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1648714993/collectinfo-2022-03-31T082317-ns_1%40172.23.120.245.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1648714993/collectinfo-2022-03-31T082317-ns_1%40172.23.121.117.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1648714993/collectinfo-2022-03-31T082317-ns_1%40172.23.123.28.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1648714993/collectinfo-2022-03-31T082317-ns_1%40172.23.96.148.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1648714993/collectinfo-2022-03-31T082317-ns_1%40172.23.96.192.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1648714993/collectinfo-2022-03-31T082317-ns_1%40172.23.96.251.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1648714993/collectinfo-2022-03-31T082317-ns_1%40172.23.96.252.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1648714993/collectinfo-2022-03-31T082317-ns_1%40172.23.96.253.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1648714993/collectinfo-2022-03-31T082317-ns_1%40172.23.97.119.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1648714993/collectinfo-2022-03-31T082317-ns_1%40172.23.97.121.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1648714993/collectinfo-2022-03-31T082317-ns_1%40172.23.97.122.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1648714993/collectinfo-2022-03-31T082317-ns_1%40172.23.97.239.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1648714993/collectinfo-2022-03-31T082317-ns_1%40172.23.99.20.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1648714993/collectinfo-2022-03-31T082317-ns_1%40172.23.99.21.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1648714993/collectinfo-2022-03-31T082317-ns_1%40172.23.99.25.zip
Attaching backup logs