[System Test][CBM] backup task failed with error - authentication error executing 'GET' request to '/api/v1/bucket/WAREHOUSE/backup' check credentials

{

  "task_name": "backup-1",

  "status": "failed",

  "start": "2021-05-05T01:00:54.497010634-07:00",

  "end": "2021-05-05T01:02:53.175572741-07:00",

  "node_runs": [

    {

      "node_id": "32902c76a67de52c427ac19e5920b6b8",

      "status": "failed",

      "start": "2021-05-05T01:00:54.558630209-07:00",

      "end": "2021-05-05T01:02:53.119133274-07:00",

      "error": "exit status 1: failed to execute cluster operations: failed to execute bucket operation for bucket 'WAREHOUSE': failed to transfer index definitions for bucket 'WAREHOUSE': failed to transfer GSI indexes: failed to get GSI index defintions: failed to get GSI index definitions: failed to execute request: authentication error executing 'GET' request to '/api/v1/bucket/WAREHOUSE/backup' check credentials",

      "progress": 0,

      "stats": {

        "error": "failed to execute cluster operations: failed to execute bucket operation for bucket 'WAREHOUSE': failed to transfer index definitions for bucket 'WAREHOUSE': failed to transfer GSI indexes: failed to get GSI index defintions: failed to get GSI index definitions: failed to execute request: authentication error executing 'GET' request to '/api/v1/bucket/WAREHOUSE/backup' check credentials",

        "stats": {

          "started_at": 1620201654756080600,

          "buckets": {

            "ITEM": {

              "estimated_total_items": 55210507,

              "total_items": 2014686,

              "total_vbuckets": 1024,

              "vbuckets_complete": 1024,

              "bytes_received": 194844388,

              "snapshot_markers_received": 1024,

              "snapshot_ends_received": 1024,

              "failover_logs_received": 1024,

              "mutations_received": 19935,

              "deletions_received": 1994751,

              "started_at": 1620201729781096400,

              "finished_at": 1620201737355237400,

              "complete": true,

              "errored": false

            },

            "NEW_ORDER": {

              "estimated_total_items": 65383751,

              "total_items": 3555945,

              "total_vbuckets": 1024,

              "vbuckets_complete": 1024,

              "bytes_received": 220298666,

              "snapshot_markers_received": 1024,

              "snapshot_ends_received": 1024,

              "failover_logs_received": 1024,

              "deletions_received": 3555945,

              "started_at": 1620201760843187700,

              "finished_at": 1620201771187130400,

              "complete": true,

              "errored": false

            },

            "bucket4": {

              "estimated_total_items": 12150114,

              "total_items": 1785941,

              "total_vbuckets": 1024,

              "vbuckets_complete": 1024,

              "bytes_received": 976705900,

              "snapshot_markers_received": 1024,

              "snapshot_ends_received": 1024,

              "failover_logs_received": 1024,

              "mutations_received": 1209000,

              "deletions_received": 576941,

              "started_at": 1620201740807808300,

              "finished_at": 1620201747400724200,

              "complete": true,

              "errored": false

            },

            "bucket5": {

              "estimated_total_items": 75622614,

              "total_items": 1755127,

              "total_vbuckets": 1024,

              "vbuckets_complete": 1024,

              "bytes_received": 975293527,

              "snapshot_markers_received": 1024,

              "snapshot_ends_received": 1024,

              "failover_logs_received": 1024,

              "mutations_received": 1209000,

              "deletions_received": 546127,

              "started_at": 1620201751072554500,

              "finished_at": 1620201757424308500,

              "complete": true,

              "errored": false

            },

            "bucket6": {

              "estimated_total_items": 24634108,

              "total_items": 651000,

              "total_vbuckets": 1024,

              "vbuckets_complete": 1024,

              "bytes_received": 32959114,

              "snapshot_markers_received": 1024,

              "snapshot_ends_received": 1024,

              "failover_logs_received": 1024,

              "deletions_received": 651000,

              "started_at": 1620201662803297000,

              "finished_at": 1620201666355451000,

              "complete": true,

              "errored": false

            },

            "bucket7": {

              "estimated_total_items": 83572364,

              "total_items": 6382445,

              "total_vbuckets": 1024,

              "vbuckets_complete": 1024,

              "bytes_received": 2221711898,

              "snapshot_markers_received": 1024,

              "snapshot_ends_received": 1024,

              "failover_logs_received": 1024,

              "mutations_received": 6165427,

              "deletions_received": 218406,

              "started_at": 1620201670071860500,

              "finished_at": 1620201694246991400,

              "complete": true,

              "errored": false

            },

            "bucket8": {

              "estimated_total_items": 453632,

              "total_items": 23529,

              "total_vbuckets": 1024,

              "vbuckets_complete": 1024,

              "bytes_received": 102400,

              "snapshot_markers_received": 1024,

              "snapshot_ends_received": 1024,

              "failover_logs_received": 1024,

              "started_at": 1620201698036574700,

              "finished_at": 1620201700383666400,

              "complete": true,

              "errored": false

            },

            "default": {

              "estimated_total_items": 171540012,

              "total_items": 6404659,

              "total_vbuckets": 1024,

              "vbuckets_complete": 1024,

              "bytes_received": 4387179203,

              "snapshot_markers_received": 1024,

              "snapshot_ends_received": 1024,

              "failover_logs_received": 1024,

              "mutations_received": 1916354,

              "deletions_received": 4495475,

              "started_at": 1620201704201033700,

              "finished_at": 1620201725036277800,

              "complete": true,

              "errored": false

            }

          },

          "complete": false

        }

      },

      "error_code": 2

    }

  ],

  "error": "exit status 1: failed to execute cluster operations: failed to execute bucket operation for bucket 'WAREHOUSE': failed to transfer index definitions for bucket 'WAREHOUSE': failed to transfer GSI indexes: failed to get GSI index defintions: failed to get GSI index definitions: failed to execute request: authentication error executing 'GET' request to '/api/v1/bucket/WAREHOUSE/backup' check credentials",

  "error_code": 2,

  "type": "BACKUP",

  "show": true

}

[2021-05-05T00:37:19-07:00, sequoiatools/couchbase-cli:7.0:617b32] rebalance -c 172.23.108.103:8091 --server-remove 172.23.104.67 -u Administrator -p password

[2021-05-05T01:03:07-07:00, sequoiatools/cmd:3a9aef] 60

[2021-05-05T01:04:13-07:00, sequoiatools/cmd:b0c596] 300

2021-05-05T01:02:51.193-07:00 (Plan) (Bucket) Transferring bucket configuration for bucket 'WAREHOUSE'

2021-05-05T01:02:51.194-07:00 (REST) (Attempt 1) (GET) Dispatching request to 'http://172.23.104.137:8091/pools/default/buckets/WAREHOUSE'

2021-05-05T01:02:51.240-07:00 (REST) (Attempt 1) (GET) (200) Received response from 'http://172.23.104.137:8091/pools/default/buckets/WAREHOUSE'

2021-05-05T01:02:51.241-07:00 (Plan) (Bucket) Successfully transferred bucket configuration for bucket 'WAREHOUSE' | {"number":69,"duration":"47.48384ms"}

2021-05-05T01:02:51.244-07:00 (Plan) (Analytics) Transferring Analytics metadata for bucket 'WAREHOUSE'

2021-05-05T01:02:51.244-07:00 (REST) (Attempt 1) (GET) Dispatching request to 'http://172.23.106.188:8095/api/v1/bucket/WAREHOUSE/backup'

2021-05-05T01:02:52.528-07:00 (REST) (Attempt 1) (GET) (200) Received response from 'http://172.23.106.188:8095/api/v1/bucket/WAREHOUSE/backup'

2021-05-05T01:02:52.540-07:00 (Plan) (Analytics) Successfully transferred Analytics metadata for bucket 'WAREHOUSE' | {"number":70,"duration":"1.296011118s"}

2021-05-05T01:02:52.543-07:00 (Plan) (Search) Transferring full text index definitions for bucket 'WAREHOUSE'

2021-05-05T01:02:52.544-07:00 (REST) (Attempt 1) (GET) Dispatching request to 'http://172.23.104.155:8094/api/v1/bucket/WAREHOUSE/backup'

2021-05-05T01:02:52.559-07:00 (REST) (Attempt 1) (GET) (200) Received response from 'http://172.23.104.155:8094/api/v1/bucket/WAREHOUSE/backup'

2021-05-05T01:02:52.560-07:00 (Plan) (Search) Successfully transferred full text index definitions for bucket 'WAREHOUSE' | {"number":71,"duration":"16.8063ms"}

2021-05-05T01:02:52.563-07:00 (Plan) (Index) Transferring GSI index definitions for bucket 'WAREHOUSE'

2021-05-05T01:02:52.564-07:00 WARN: (Auth) (Service) Could not get credentials for host 'http://172.23.104.67:9102': could not get credentials from the backup service: rpc error: code = Internal desc = failed to get credentials from cbauth: Unable to find given hostport in cbauth database: `172.23.104.67:9102' -- auth.BaseProvider.GetCredentials() at base_provider.go:81

2021-05-05T01:02:52.564-07:00 (REST) (Attempt 1) (GET) Dispatching request to 'http://172.23.104.67:9102/api/v1/bucket/WAREHOUSE/backup'

2021-05-05T01:02:52.568-07:00 (REST) (Attempt 1) (GET) (401) Received response from 'http://172.23.104.67:9102/api/v1/bucket/WAREHOUSE/backup'

2021-05-05T01:02:52.568-07:00 WARN: (REST) (Attempt 1) (GET) Request to endpoint '/api/v1/bucket/WAREHOUSE/backup' failed with status code 401 -- rest.(*Request).Execute() at request.go:143

2021-05-05T01:02:52.568-07:00 (Cmd) Error backing up cluster: failed to execute cluster operations: failed to execute bucket operation for bucket 'WAREHOUSE': failed to transfer index definitions for bucket 'WAREHOUSE': failed to transfer GSI indexes: failed to get GSI index defintions: failed to get GSI index definitions: failed to execute request: authentication error executing 'GET' request to '/api/v1/bucket/WAREHOUSE/backup' check credentials