[System Test][Backup Service] gosecrets invoked oom-killer

{

  "task_name": "backup-1",

  "status": "failed",

  "start": "2021-12-09T16:00:56.046059047-08:00",

  "end": "2021-12-09T17:21:05.86033512-08:00",

  "node_runs": [

    {

      "node_id": "a9f4f45f4650e1e32e7c46450c91dfcf",

      "status": "failed",

      "start": "2021-12-09T16:53:36.965618404-08:00",

      "end": "2021-12-09T17:21:05.79476143-08:00",

      "error": "signal: killed",

      "progress": 71.05401613468958,

      "stats": {

        "id": "afa57897-7907-49fb-a557-96b0ebff18f8",

        "current_transfer": 1,

        "total_transfers": 1,

        "transfers": [

          {

            "description": "Backing up to 2021-12-09T16_54_56.779644641-08_00",

            "stats": {

              "started_at": 1639097695191315700,

              "buckets": {

                "N1QL_SYSTEM_BUCKET": {

                  "estimated_total_items": 17237,

                  "total_items": 551,

                  "total_vbuckets": 1024,

                  "vbuckets_complete": 1024,

                  "bytes_received": 579438,

                  "snapshot_markers_received": 1024,

                  "failover_logs_received": 1024,

                  "mutations_received": 280,

                  "deletions_received": 271,

                  "started_at": 1639098089502838000,

                  "finished_at": 1639098094688621300,

                  "complete": true

                },

                "bucket1": {

                  "estimated_total_items": 121618590,

                  "total_items": 6054730,

                  "total_vbuckets": 1024,

                  "vbuckets_complete": 838,

                  "bytes_received": 3586775948,

                  "snapshot_markers_received": 1024,

                  "failover_logs_received": 1024,

                  "mutations_received": 2831274,

                  "deletions_received": 2974518,

                  "started_at": 1639099140496877600

                },

                "bucket2": {

                  "estimated_total_items": 181541734,

                  "total_items": 7599670,

                  "total_vbuckets": 1024,

                  "vbuckets_complete": 1024,

                  "bytes_received": 512800364,

                  "snapshot_markers_received": 1024,

                  "failover_logs_received": 1024,

                  "mutations_received": 1012243,

                  "deletions_received": 6596174,

                  "started_at": 1639098232947880200,

                  "finished_at": 1639098253120513000,

                  "complete": true

                },

                "bucket3": {

                  "estimated_total_items": 168065094,

                  "total_items": 3783968,

                  "total_vbuckets": 1024,

                  "vbuckets_complete": 1024,

                  "bytes_received": 358567410,

                  "snapshot_markers_received": 1024,

                  "failover_logs_received": 1024,

                  "mutations_received": 25329,

                  "deletions_received": 3762588,

                  "started_at": 1639098392036248600,

                  "finished_at": 1639098402561738800,

                  "complete": true

                },

                "bucket4": {

                  "estimated_total_items": 7174397,

                  "total_items": 1838703,

                  "total_vbuckets": 1024,

                  "vbuckets_complete": 1024,

                  "bytes_received": 1325580160,

                  "snapshot_markers_received": 1024,

                  "failover_logs_received": 1024,

                  "mutations_received": 1675953,

                  "deletions_received": 162750,

                  "started_at": 1639098548752666000,

                  "finished_at": 1639098585815213300,

                  "complete": true

                },

                "bucket5": {

                  "estimated_total_items": 23808751,

                  "total_items": 1840719,

                  "total_vbuckets": 1024,

                  "vbuckets_complete": 1024,

                  "bytes_received": 1443334484,

                  "snapshot_markers_received": 1024,

                  "failover_logs_received": 1024,

                  "mutations_received": 1836939,

                  "deletions_received": 1764,

                  "started_at": 1639097702495704800,

                  "finished_at": 1639097742509096700,

                  "complete": true

                },

                "bucket6": {

                  "estimated_total_items": 18909881,

                  "total_items": 1838703,

                  "total_vbuckets": 1024,

                  "vbuckets_complete": 1024,

                  "bytes_received": 1325036928,

                  "snapshot_markers_received": 1024,

                  "failover_logs_received": 1024,

                  "mutations_received": 1675953,

                  "deletions_received": 162750,

                  "started_at": 1639097886092483600,

                  "finished_at": 1639097928120881700,

                  "complete": true

                },

                "default": {

                  "estimated_total_items": 735460661,

                  "total_items": 17704370,

                  "total_vbuckets": 1024,

                  "vbuckets_complete": 1024,

                  "bytes_received": 12361200184,

                  "snapshot_markers_received": 1024,

                  "failover_logs_received": 1024,

                  "mutations_received": 3839709,

                  "deletions_received": 13905228,

                  "started_at": 1639098724133792500,

                  "finished_at": 1639098977534237400,

                  "complete": true

                }

              }

            },

            "progress": 71.05401613468958,

            "eta": "2021-12-09T17:31:38.167998493-08:00"

          }

        ],

        "progress": 71.05401613468958,

        "eta": "2021-12-09T17:31:38.167998493-08:00"

      },

      "error_code": 2

    }

  ],

  "error": "signal: killed",

  "error_code": 2,

  "type": "BACKUP",

  "show": true

}

2021-12-09T17:21:03.114-08:00 INFO (Janitor) Running storage access check

2021-12-09T17:21:03.297-08:00 INFO (Janitor) Running storage access check on state active

2021-12-09T17:21:03.320-08:00 INFO (GRPC Manager) Global storage location check {"req": "location:\"/data/archive/.cbbs-86ff6a60dc5018c1ad3173c9a32509b0/node-a9f4f45f4650e1e32e7c46450c91dfcf-1639099263312711902\"  sourceNode:\"a9f4f45f4650e1e32e7c46450c91dfcf\""}

2021-12-09T17:21:03.781-08:00 INFO (GRPC Manager) Global storage location check {"req": "location:\"centos1/.cbbs-86ff6a60dc5018c1ad3173c9a32509b0/node-a9f4f45f4650e1e32e7c46450c91dfcf-1639099263340158912\"  sourceNode:\"a9f4f45f4650e1e32e7c46450c91dfcf\"  cloud:true  credentialsName:\"c1fd8397-0788-4161-b603-0a14fbbd57cf\"  bucket:\"longevity_testing\"  provider:\"gcp\""}

2021-12-09T17:21:05.379-08:00 INFO (GRPC Manager) Global storage location check {"req": "location:\"longevity-testing/centos1/.cbbs-86ff6a60dc5018c1ad3173c9a32509b0/node-a9f4f45f4650e1e32e7c46450c91dfcf-1639099264757241940\"  sourceNode:\"a9f4f45f4650e1e32e7c46450c91dfcf\"  cloud:true  credentialsName:\"9d6afa69-e3a0-4790-b88b-4ef9b69a377d\"  bucket:\"backup-service-testing\"  provider:\"aws\"  region:\"eu-west-2\""}

2021-12-09T17:21:05.793-08:00 INFO (Worker) Command exited {"err": "signal: killed", "cluster": "self", "repositoryID": "my_repo_gcp", "state": "active", "taskName": "backup-1"}

2021-12-09T17:21:05.794-08:00 DEBUG (REST) (Attempt 1) (POST) Dispatching request to 'http://127.0.0.1:8091/_event'

2021-12-09T17:21:05.794-08:00 WARN (Worker) Task failed {"cluster": "self", "repositoryID": "my_repo_gcp", "state": "active", "taskName": "backup-1"

########## Cluster config ##################

######  fts : 2 ===== > [172.23.104.155:8091 172.23.96.148:8091]  ###########

######  eventing : 2 ===== > [172.23.104.5:8091 172.23.98.135:8091]  ###########

######  kv : 11 ===== > [172.23.104.67:8091 172.23.104.69:8091 172.23.108.103:8091 172.23.96.251:8091 172.23.97.119:8091 172.23.97.121:8091 172.23.97.122:8091 172.23.97.239:8091 172.23.99.20:8091 172.23.99.21:8091 172.23.99.25:8091]  ###########

######  index : 6 ===== > [172.23.104.70:8091 172.23.120.107:8091 172.23.120.98:8091 172.23.121.117:8091 172.23.96.252:8091 172.23.96.253:8091]  ###########

######  cbas : 1 ===== > [172.23.106.188:8091]  ###########

######  backup : 1 ===== > [172.23.123.28:8091]  ###########

######  n1ql : 2 ===== > [172.23.104.137:8091 172.23.99.11:8091]  ###########

[2021-12-09T15:58:56-08:00, sequoiatools/couchbase-cli:7.1:d5e8a1] rebalance -c 172.23.108.103:8091 -u Administrator -p password

[2021-12-09T16:01:13-08:00, sequoiatools/cmd:604913] 60

[2021-12-09T16:02:19-08:00, sequoiatools/cmd:f7919a] 300

[2021-12-09T16:07:59-08:00, sequoiatools/couchbase-cli:7.1:ebb25b] rebalance -c 172.23.108.103:8091 --server-remove 172.23.104.5 -u Administrator -p password

[2021-12-09T16:08:58-08:00, sequoiatools/cmd:c62fef] 60

[2021-12-09T16:10:04-08:00, sequoiatools/cmd:c5bed7] 300

[2021-12-09T16:17:05-08:00, sequoiatools/couchbase-cli:7.1:80e96a] server-add -c 172.23.108.103:8091 --server-add https://172.23.104.5 -u Administrator -p password --server-add-username Administrator --server-add-password password --services eventing

[2021-12-09T16:17:20-08:00, sequoiatools/couchbase-cli:7.1:814c90] rebalance -c 172.23.108.103:8091 --server-remove 172.23.104.137 -u Administrator -p password

[2021-12-09T16:19:58-08:00, sequoiatools/cmd:d9091f] 60

########## Cluster config ##################

######  fts : 2 ===== > [172.23.104.155:8091 172.23.96.148:8091]  ###########

######  eventing : 2 ===== > [172.23.104.5:8091 172.23.98.135:8091]  ###########

######  index : 6 ===== > [172.23.104.67:8091 172.23.104.70:8091 172.23.120.98:8091 172.23.121.117:8091 172.23.96.252:8091 172.23.96.253:8091]  ###########

######  kv : 11 ===== > [172.23.104.69:8091 172.23.106.100:8091 172.23.108.103:8091 172.23.96.251:8091 172.23.97.119:8091 172.23.97.121:8091 172.23.97.122:8091 172.23.97.239:8091 172.23.99.20:8091 172.23.99.21:8091 172.23.99.25:8091]  ###########

######  cbas : 1 ===== > [172.23.106.188:8091]  ###########

######  n1ql : 2 ===== > [172.23.120.107:8091 172.23.99.11:8091]  ###########

######  backup : 1 ===== > [172.23.123.28:8091]  ###########

[2021-12-09T16:21:05-08:00, sequoiatools/cmd:0b738f] 600

→  parsed tests/analytics/cheshirecat/test_analytics_integration_scale3.yml

→  parsed providers/file/centos_pine.yml

→  parsed providers/file/centos_pine.yml

[pull] sequoiatools/couchbase-cli:7.1

Test cycle started: 1

→  parsed tests/templates/kv.yml

→  parsed tests/templates/vegeta.yml

→  parsed tests/templates/analytics.yml

→  parsed tests/templates/rebalance.yml

[2021-12-09T16:32:46-08:00, sequoiatools/couchbase-cli:7.1:d6fae0] server-add -c 172.23.108.103:8091 --server-add https://172.23.97.242 -u Administrator -p password --server-add-username Administrator --server-add-password password --services analytics

[2021-12-09T16:33:00-08:00, sequoiatools/couchbase-cli:7.1:2f609f] rebalance -c 172.23.108.103:8091 -u Administrator -p password

[2021-12-09T16:39:57-08:00, sequoiatools/cmd:934935] 60

[2021-12-09T16:41:03-08:00, sequoiatools/cmd:f1b435] 30

[2021-12-09T16:42:14-08:00, sequoiatools/couchbase-cli:7.1:60f46f] rebalance -c 172.23.108.103:8091 --server-remove 172.23.97.242 -u Administrator -p password

[2021-12-09T16:51:37-08:00, sequoiatools/cmd:f5a535] 60

[2021-12-09T16:52:42-08:00, sequoiatools/cmd:1b2053] 30

[2021-12-09T16:55:24-08:00, sequoiatools/couchbase-cli:7.1:7eab8a] server-add -c 172.23.108.103:8091 --server-add https://172.23.97.242 -u Administrator -p password --server-add-username Administrator --server-add-password password --services analytics

[2021-12-09T16:55:38-08:00, sequoiatools/couchbase-cli:7.1:576f0d] rebalance -c 172.23.108.103:8091 --server-remove <node_not_found> -u Administrator -p password

→  

Error occurred on container - sequoiatools/couchbase-cli:7.1:[rebalance -c 172.23.108.103:8091 --server-remove <node_not_found> -u Administrator -p password]

docker logs 576f0d

docker start 576f0d

FERROR: Some nodes specified to be removed are not part of the cluster

[2021-12-09T16:55:45-08:00, sequoiatools/cmd:184d45] 60

[2021-12-09T16:56:50-08:00, sequoiatools/cmd:5a9e9f] 300

########## Cluster config ##################

######  cbas : 2 ===== > [172.23.106.188:8091 172.23.97.242:8091]  ###########

######  n1ql : 2 ===== > [172.23.120.107:8091 172.23.99.11:8091]  ###########

######  backup : 1 ===== > [172.23.123.28:8091]  ###########

######  fts : 2 ===== > [172.23.104.155:8091 172.23.96.148:8091]  ###########

######  eventing : 2 ===== > [172.23.104.5:8091 172.23.98.135:8091]  ###########

######  index : 6 ===== > [172.23.104.67:8091 172.23.104.70:8091 172.23.120.98:8091 172.23.121.117:8091 172.23.96.252:8091 172.23.96.253:8091]  ###########

######  kv : 11 ===== > [172.23.104.69:8091 172.23.106.100:8091 172.23.108.103:8091 172.23.96.251:8091 172.23.97.119:8091 172.23.97.121:8091 172.23.97.122:8091 172.23.97.239:8091 172.23.99.20:8091 172.23.99.21:8091 172.23.99.25:8091]  ###########

[2021-12-09T17:01:59-08:00, sequoiatools/cmd:518772] 600

[2021-12-09T17:12:04-08:00, danihodovic/vegeta:ce45e5] bash -c echo GET "http://Administrator:password@172.23.108.103:8092/default/_design/scale/_view/stats?limit=30&stale=update_after&connection_timeout=60000" | vegeta attack -duration=0 -rate=10> results.bin && vegeta report -inputs=results.bin > results.txt && vegeta report -inputs=results.bin -reporter=plot > plot.html

[2021-12-09T17:12:09-08:00, danihodovic/vegeta:85b668] bash -c echo GET "http://Administrator:password@172.23.106.100:8092/default/_design/scale/_view/array?limit=30&stale=update_after&connection_timeout=60000" | vegeta attack -duration=0 -rate=10> results.bin && vegeta report -inputs=results.bin > results.txt && vegeta report -inputs=results.bin -reporter=plot > plot.html

[2021-12-09T17:12:15-08:00, danihodovic/vegeta:80059e] bash -c echo GET "http://Administrator:password@172.23.97.122:8092/default/_design/scale/_view/padd?limit=30&stale=update_after&connection_timeout=60000" | vegeta attack -duration=0 -rate=10> results.bin && vegeta report -inputs=results.bin > results.txt && vegeta report -inputs=results.bin -reporter=plot > plot.html

[2021-12-09T17:12:19-08:00, appropriate/curl:4ed3c4] -s http://Administrator:password@172.23.108.103:8091/pools/default/remoteClusters

→  parsed tests/eventing/CC/test_eventing_rebalance_integration.yml

→  parsed providers/file/centos_pine.yml

→  parsed providers/file/centos_pine.yml

[pull] sequoiatools/couchbase-cli:7.1

Test cycle started: 1

→  parsed tests/templates/kv.yml

→  parsed tests/templates/vegeta.yml

→  parsed tests/templates/rebalance.yml

[2021-12-09T17:12:26-08:00, sequoiatools/eventing:7.0:d63fb8] eventing_helper.py -i 172.23.98.135 -u Administrator -p password -o pause

[2021-12-09T17:12:33-08:00, sequoiatools/eventing:7.0:985a27] eventing_helper.py -i 172.23.98.135 -u Administrator -p password -o wait_for_state --state paused

########## Cluster config ##################

######  backup : 1 ===== > [172.23.123.28:8091]  ###########

######  fts : 2 ===== > [172.23.104.155:8091 172.23.96.148:8091]  ###########

######  eventing : 2 ===== > [172.23.104.5:8091 172.23.98.135:8091]  ###########

######  index : 6 ===== > [172.23.104.67:8091 172.23.104.70:8091 172.23.120.98:8091 172.23.121.117:8091 172.23.96.252:8091 172.23.96.253:8091]  ###########

######  kv : 11 ===== > [172.23.104.69:8091 172.23.106.100:8091 172.23.108.103:8091 172.23.96.251:8091 172.23.97.119:8091 172.23.97.121:8091 172.23.97.122:8091 172.23.97.239:8091 172.23.99.20:8091 172.23.99.21:8091 172.23.99.25:8091]  ###########

######  cbas : 2 ===== > [172.23.106.188:8091 172.23.97.242:8091]  ###########

######  n1ql : 2 ===== > [172.23.120.107:8091 172.23.99.11:8091]  ###########

[2021-12-09T17:13:01-08:00, sequoiatools/pillowfight:7.0:dc920c] -U couchbase://172.23.108.103/default?select_bucket=true -I 3000 -B 300 -t 4 -c 100 -P password

→  parsed tests/2i/neo/test_idx_neo_integration.yml

→  parsed providers/file/centos_pine.yml

→  parsed providers/file/centos_pine.yml

[pull] sequoiatools/couchbase-cli:7.1

Test cycle started: 1

→  parsed tests/templates/kv.yml

→  parsed tests/templates/n1ql.yml

→  parsed tests/templates/rebalance.yml

[2021-12-09T17:14:12-08:00, sequoiatools/couchbase-cli:7.1:99c168] server-add -c 172.23.108.103:8091 --server-add https://172.23.97.242 -u Administrator -p password --server-add-username Administrator --server-add-password password --services query

→  

Error occurred on container - sequoiatools/couchbase-cli:7.1:[server-add -c 172.23.108.103:8091 --server-add https://172.23.97.242 -u Administrator -p password --server-add-username Administrator --server-add-password password --services query]

docker logs 99c168

docker start 99c168

=ERROR: Prepare join failed. Node is already part of cluster.

[2021-12-09T17:14:18-08:00, sequoiatools/couchbase-cli:7.1:4543e0] rebalance -c 172.23.108.103:8091 -u Administrator -p password

[2021-12-09T17:21:14-08:00, sequoiatools/cmd:5fbc6c] 60

[2021-12-09T17:22:20-08:00, sequoiatools/cmd:065848] 300