[System Test][Backup Service] post to ensureManifest API failed with context exceeded deadline error

{

  "task_name": "backup-1",

  "status": "failed",

  "start": "2021-06-24T04:00:52.582699858-07:00",

  "end": "2021-06-24T04:06:14.357283886-07:00",

  "node_runs": [

    {

      "node_id": "301d9c46c75f781a445f5ccc86627f95",

      "status": "failed",

      "start": "2021-06-24T04:00:52.680763742-07:00",

      "end": "2021-06-24T04:06:14.272473317-07:00",

      "error": "exit status 1: failed to execute cluster operations: failed to execute bucket operation for bucket 'bucket9': failed to transfer bucket data for bucket 'bucket9': failed to transfer key value data: failed to update source scope/collection mappings: failed to get source collection manifest: failed to wait for manifest id 7082: failed to execute request: http client failed to dispatch/receive request/response: Post \"http://172.23.104.137:8091/pools/default/buckets/bucket9/scopes/%40ensureManifest/1baa\": context deadline exceeded",

      "progress": 0,

      "stats": {

        "error": "failed to execute cluster operations: failed to execute bucket operation for bucket 'bucket9': failed to transfer bucket data for bucket 'bucket9': failed to transfer key value data: failed to update source scope/collection mappings: failed to get source collection manifest: failed to wait for manifest id 7082: failed to execute request: http client failed to dispatch/receive request/response: Post \"http://172.23.104.137:8091/pools/default/buckets/bucket9/scopes/%40ensureManifest/1baa\": context deadline exceeded",

        "stats": {

          "started_at": 1624532453005698000,

          "buckets": {

            "ITEM": {

              "estimated_total_items": 1256546025,

              "total_items": 6236770,

              "total_vbuckets": 1024,

              "vbuckets_complete": 1024,

              "bytes_received": 598892232,

              "snapshot_markers_received": 1024,

              "snapshot_ends_received": 1024,

              "failover_logs_received": 1024,

              "mutations_received": 58837,

              "deletions_received": 6178993,

              "started_at": 1624532579841792800,

              "finished_at": 1624532608806792700,

              "complete": true,

              "errored": false

            },

            "WAREHOUSE": {

              "estimated_total_items": 1115623764,

              "total_items": 9464205,

              "total_vbuckets": 1024,

              "vbuckets_complete": 1024,

              "bytes_received": 4573243797,

              "snapshot_markers_received": 1024,

              "snapshot_ends_received": 1024,

              "failover_logs_received": 1024,

              "mutations_received": 3352717,

              "deletions_received": 6140212,

              "started_at": 1624532612596392200,

              "finished_at": 1624532661774220800,

              "complete": true,

              "errored": false

            },

            "bucket5": {

              "estimated_total_items": 261590869,

              "total_items": 1619351,

              "total_vbuckets": 1024,

              "vbuckets_complete": 1024,

              "bytes_received": 874631011,

              "snapshot_markers_received": 1024,

              "snapshot_ends_received": 1024,

              "failover_logs_received": 1024,

              "mutations_received": 1078459,

              "deletions_received": 540892,

              "started_at": 1624532667058550300,

              "finished_at": 1624532688710052000,

              "complete": true,

              "errored": false

            },

            "bucket6": {

              "estimated_total_items": 178110187,

              "total_items": 1653282,

              "total_vbuckets": 1024,

              "vbuckets_complete": 1024,

              "bytes_received": 882300612,

              "snapshot_markers_received": 1024,

              "snapshot_ends_received": 1024,

              "failover_logs_received": 1024,

              "mutations_received": 1086240,

              "deletions_received": 567042,

              "started_at": 1624532692715339300,

              "finished_at": 1624532708478172200,

              "complete": true,

              "errored": false

            },

            "bucket7": {

              "estimated_total_items": 775626571,

              "total_items": 239588,

              "total_vbuckets": 1024,

              "vbuckets_complete": 1024,

              "bytes_received": 18500090,

              "snapshot_markers_received": 1024,

              "snapshot_ends_received": 1024,

              "failover_logs_received": 1024,

              "mutations_received": 13,

              "deletions_received": 239575,

              "started_at": 1624532464043596800,

              "finished_at": 1624532468207193900,

              "complete": true,

              "errored": false

            },

            "bucket8": {

              "estimated_total_items": 7993614,

              "total_items": 34164,

              "total_vbuckets": 1024,

              "vbuckets_complete": 1024,

              "bytes_received": 131984,

              "snapshot_markers_received": 1024,

              "snapshot_ends_received": 1024,

              "failover_logs_received": 1024,

              "started_at": 1624532471975479000,

              "finished_at": 1624532474707847000,

              "complete": true,

              "errored": false

            },

            "default": {

              "estimated_total_items": 4234421084,

              "total_items": 13457224,

              "total_vbuckets": 1024,

              "vbuckets_complete": 1024,

              "bytes_received": 12777513503,

              "snapshot_markers_received": 1024,

              "snapshot_ends_received": 1024,

              "failover_logs_received": 1024,

              "mutations_received": 2235205,

              "deletions_received": 11240675,

              "started_at": 1624532478862219800,

              "finished_at": 1624532576001258200,

              "complete": true,

              "errored": false

            }

          },

          "complete": false

        }

      },

      "error_code": 2

    }

  ],

  "error": "exit status 1: failed to execute cluster operations: failed to execute bucket operation for bucket 'bucket9': failed to transfer bucket data for bucket 'bucket9': failed to transfer key value data: failed to update source scope/collection mappings: failed to get source collection manifest: failed to wait for manifest id 7082: failed to execute request: http client failed to dispatch/receive request/response: Post \"http://172.23.104.137:8091/pools/default/buckets/bucket9/scopes/%40ensureManifest/1baa\": context deadline exceeded",

  "error_code": 2,

  "type": "BACKUP",

  "show": true

}