[System test][CBM] backup task failed with error "Analytics Service is temporarily unavailable" while rebalance was going on

{

  "task_name": "backup-1",

  "status": "failed",

  "start": "2022-03-31T01:00:41.471941428-07:00",

  "end": "2022-03-31T01:02:45.225100363-07:00",

  "node_runs": [

    {

      "node_id": "bfa1066b243e98ef63aa76c81b9ce123",

      "status": "failed",

      "start": "2022-03-31T01:00:41.736847177-07:00",

      "end": "2022-03-31T01:02:44.98071655-07:00",

      "error": "exit status 1: failed to execute cluster operations: failed to execute bucket operation for bucket 'bucket7': failed to transfer Analytics for bucket 'bucket7': failed to get Analytics metadata: failed to get analytics metadata: failed to execute request: failed to execute request: exhausted retry count after 3 retries, last error: internal server error executing 'GET' request to '/api/v1/bucket/bucket7/backup': {\n\t\"version\": 2,\n\t\"requestID\": \"6596b60a-45f1-473b-9b95-527377cc7a76\",\n\t\"errors\": [{ \n\t\t\"code\": 23000,\t\t\"msg\": \"Analytics Service is temporarily unavailable\"\t} \n\t],\n\t\"status\": \"fatal\",\n\t\"metrics\": {\n\t\t\"elapsedTime\": \"8.737995ms\",\n\t\t\"executionTime\": \"2.332567ms\",\n\t\t\"resultCount\": 0,\n\t\t\"resultSize\": 0,\n\t\t\"processedObjects\": 0,\n\t\t\"errorCount\": 1\n\t}\n}\n",

      "progress": 0,

      "stats": {

        "error": "failed to execute cluster operations: failed to execute bucket operation for bucket 'bucket7': failed to transfer Analytics for bucket 'bucket7': failed to get Analytics metadata: failed to get analytics metadata: failed to execute request: failed to execute request: exhausted retry count after 3 retries, last error: internal server error executing 'GET' request to '/api/v1/bucket/bucket7/backup': {\n\t\"version\": 2,\n\t\"requestID\": \"6596b60a-45f1-473b-9b95-527377cc7a76\",\n\t\"errors\": [{ \n\t\t\"code\": 23000,\t\t\"msg\": \"Analytics Service is temporarily unavailable\"\t} \n\t],\n\t\"status\": \"fatal\",\n\t\"metrics\": {\n\t\t\"elapsedTime\": \"8.737995ms\",\n\t\t\"executionTime\": \"2.332567ms\",\n\t\t\"resultCount\": 0,\n\t\t\"resultSize\": 0,\n\t\t\"processedObjects\": 0,\n\t\t\"errorCount\": 1\n\t}\n}\n",

        "stats": {

          "started_at": 1648713642861233200,

          "buckets": {

            "N1QL_SYSTEM_BUCKET": {

              "estimated_total_items": 13601,

              "total_items": 681,

              "total_vbuckets": 1024,

              "vbuckets_complete": 1024,

              "bytes_received": 508298,

              "snapshot_markers_received": 1024,

              "failover_logs_received": 1024,

              "mutations_received": 219,

              "deletions_received": 462,

              "started_at": 1648713660297268500,

              "finished_at": 1648713663897422300,

              "complete": true

            },

            "bucket1": {

              "estimated_total_items": 303701253,

              "total_items": 4497330,

              "total_vbuckets": 1024,

              "vbuckets_complete": 1024,

              "bytes_received": 3504944425,

              "snapshot_markers_received": 1024,

              "failover_logs_received": 1024,

              "mutations_received": 2881526,

              "deletions_received": 1619117,

              "started_at": 1648713668390584000,

              "finished_at": 1648713685151503000,

              "complete": true

            },

            "bucket2": {

              "estimated_total_items": 452241217,

              "total_items": 6949797,

              "total_vbuckets": 1024,

              "vbuckets_complete": 1024,

              "bytes_received": 468143838,

              "snapshot_markers_received": 1024,

              "failover_logs_received": 1024,

              "mutations_received": 949825,

              "deletions_received": 6001249,

              "started_at": 1648713737869911300,

              "finished_at": 1648713752476223000,

              "complete": true

            },

            "bucket3": {

              "estimated_total_items": 479400782,

              "total_items": 3290309,

              "total_vbuckets": 1024,

              "vbuckets_complete": 1024,

              "bytes_received": 311496061,

              "snapshot_markers_received": 1024,

              "failover_logs_received": 1024,

              "mutations_received": 29980,

              "deletions_received": 3244226,

              "started_at": 1648713688395884300,

              "finished_at": 1648713698094012700,

              "complete": true

            },

            "bucket4": {

              "estimated_total_items": 27300963,

              "total_items": 1817654,

              "total_vbuckets": 1024,

              "vbuckets_complete": 1024,

              "bytes_received": 1428666934,

              "snapshot_markers_received": 1024,

              "failover_logs_received": 1024,

              "mutations_received": 1817654,

              "started_at": 1648713703646307600,

              "finished_at": 1648713712436376600,

              "complete": true

            },

            "bucket5": {

              "estimated_total_items": 41910982,

              "total_items": 1818181,

              "total_vbuckets": 1024,

              "vbuckets_complete": 1024,

              "bytes_received": 1309530609,

              "snapshot_markers_received": 1024,

              "failover_logs_received": 1024,

              "mutations_received": 1655431,

              "deletions_received": 162750,

              "started_at": 1648713756765534200,

              "finished_at": 1648713763967859200,

              "complete": true

            },

            "bucket6": {

              "estimated_total_items": 50324648,

              "total_items": 1818181,

              "total_vbuckets": 1024,

              "vbuckets_complete": 1024,

              "bytes_received": 1309961495,

              "snapshot_markers_received": 1024,

              "failover_logs_received": 1024,

              "mutations_received": 1655431,

              "deletions_received": 162750,

              "started_at": 1648713717641000000,

              "finished_at": 1648713725528399000,

              "complete": true

            },

            "bucket8": {

              "estimated_total_items": 1738832,

              "total_items": 44859,

              "total_vbuckets": 1024,

              "vbuckets_complete": 1024,

              "bytes_received": 80944,

              "snapshot_markers_received": 1024,

              "failover_logs_received": 1024,

              "started_at": 1648713730707312000,

              "finished_at": 1648713733437451500,

              "complete": true

            },

            "bucket9": {

              "estimated_total_items": 1700460,

              "total_items": 18476,

              "total_vbuckets": 1024,

              "vbuckets_complete": 1024,

              "bytes_received": 80944,

              "snapshot_markers_received": 1024,

              "failover_logs_received": 1024,

              "started_at": 1648713652472698000,

              "finished_at": 1648713655555433000,

              "complete": true

            }

          }

        }

      },

      "error_code": 2

    }

  ],

  "error": "exit status 1: failed to execute cluster operations: failed to execute bucket operation for bucket 'bucket7': failed to transfer Analytics for bucket 'bucket7': failed to get Analytics metadata: failed to get analytics metadata: failed to execute request: failed to execute request: exhausted retry count after 3 retries, last error: internal server error executing 'GET' request to '/api/v1/bucket/bucket7/backup': {\n\t\"version\": 2,\n\t\"requestID\": \"6596b60a-45f1-473b-9b95-527377cc7a76\",\n\t\"errors\": [{ \n\t\t\"code\": 23000,\t\t\"msg\": \"Analytics Service is temporarily unavailable\"\t} \n\t],\n\t\"status\": \"fatal\",\n\t\"metrics\": {\n\t\t\"elapsedTime\": \"8.737995ms\",\n\t\t\"executionTime\": \"2.332567ms\",\n\t\t\"resultCount\": 0,\n\t\t\"resultSize\": 0,\n\t\t\"processedObjects\": 0,\n\t\t\"errorCount\": 1\n\t}\n}\n",

  "error_code": 2,

  "type": "BACKUP",

  "show": true

} 

 backup : 1 ===== > [172.23.104.69:8091] ########### ###### kv : 11 ===== > [172.23.105.107:8091 172.23.106.100:8091 172.23.108.103:8091 172.23.121.117:8091 172.23.96.253:8091 172.23.97.119:8091 172.23.97.121:8091 172.23.97.122:8091 172.23.99.20:8091 172.23.99.21:8091 172.23.99.25:8091] ########### ###### fts : 2 ===== > [172.23.105.111:8091 172.23.96.148:8091] ########### ###### cbas : 3 ===== > [172.23.105.168:8091 172.23.106.188:8091 172.23.97.239:8091] ########### ###### n1ql : 2 ===== > [172.23.104.137:8091 172.23.99.11:8091] ########### ###### index : 6 ===== > [172.23.104.155:8091 172.23.120.107:8091 172.23.120.245:8091 172.23.123.28:8091 172.23.96.251:8091 172.23.96.252:8091] ########### ###### eventing : 3 ===== > [172.23.104.5:8091 172.23.104.67:8091 172.23.96.192:8091] ###########