[System Test] Rebalance exited with reason "bucket_cleanup_failed"

-test tests/integration/cheshirecat/test_cheshirecat_kv_gsi_coll_xdcr_backup_sgw_fts_itemct_txns_eventing_cbas_scale3.yml -scope tests/integration/cheshirecat/scope_cheshirecat_with_backup.yml

[2021-08-26T17:00:16-07:00, sequoiatools/couchbase-cli:7.0:564b03] rebalance -c 172.23.97.74:8091 -u Administrator -p password

→  

Error occurred on container - sequoiatools/couchbase-cli:7.0:[rebalance -c 172.23.97.74:8091 -u Administrator -p password]

docker logs 564b03

docker start 564b03

*Unable to display progress bar on this os

JERROR: Rebalance failed. See logs for detailed reason. You can try again.

[2021-08-26T17:00:43-07:00, sequoiatools/cmd:81e726] 60

[2021-08-26T17:01:50-07:00, sequoiatools/cmd:ad54a4] 30

[2021-08-26T17:04:01-07:00, sequoiatools/couchbase-cli:7.0:7f925b] rebalance -c 172.23.97.74:8091 --server-remove 172.23.120.74 -u Administrator -p password

→  

Error occurred on container - sequoiatools/couchbase-cli:7.0:[rebalance -c 172.23.97.74:8091 --server-remove 172.23.120.74 -u Administrator -p password]

docker logs 7f925b

docker start 7f925b

*Unable to display progress bar on this os

JERROR: Rebalance failed. See logs for detailed reason. You can try again.

[rebalance:error,2021-08-26T17:00:35.559-07:00,ns_1@172.23.97.74:<0.26694.1851>:ns_rebalancer:maybe_cleanup_old_buckets:941]Failed to cleanup old buckets on node 'ns_1@172.23.123.26': {badrpc,

                                                             {'EXIT',timeout}}

[ns_server:info,2021-08-26T17:00:35.561-07:00,ns_1@172.23.97.74:rebalance_agent<0.22070.0>:rebalance_agent:handle_down:290]Rebalancer process <0.26694.1851> died (reason {buckets_cleanup_failed,

                                                ['ns_1@172.23.123.26']}).

[ns_server:debug,2021-08-26T17:00:35.561-07:00,ns_1@172.23.97.74:leader_activities<0.23455.0>:leader_activities:handle_activity_down:505]Activity terminated with reason {shutdown,

                                 {async_died,

                                  {raised,

                                   {exit,

                                    {buckets_cleanup_failed,

                                     ['ns_1@172.23.123.26']},

                                    [{ns_rebalancer,rebalance_kv,4,

                                      [{file,"src/ns_rebalancer.erl"},

                                       {line,602}]},

                                     {ns_rebalancer,rebalance_body,5,

                                      [{file,"src/ns_rebalancer.erl"},

                                       {line,556}]},

                                     {async,'-async_init/4-fun-1-',3,

                                      [{file,"src/async.erl"},

                                       {line,191}]}]}}}}. Activity:

{activity,<0.29036.1851>,#Ref<0.3648888551.3028549638.183159>,default,

          <<"5733b46753bb092077bb9e90f53979c7">>,

          [rebalance],

          majority,[]}

[error_logger:error,2021-08-26T17:00:35.562-07:00,ns_1@172.23.97.74:<0.26198.1851>:ale_error_logger_handler:do_log:101]

=========================CRASH REPORT=========================

  crasher:

    initial call: erlang:apply/2

    pid: <0.26198.1851>

    registered_name: []

    exception exit: {buckets_cleanup_failed,['ns_1@172.23.123.26']}

      in function  ns_rebalancer:rebalance_kv/4 (src/ns_rebalancer.erl, line 602)

      in call from ns_rebalancer:rebalance_body/5 (src/ns_rebalancer.erl, line 556)

      in call from async:'-async_init/4-fun-1-'/3 (src/async.erl, line 191)

    ancestors: [<0.23486.0>,ns_orchestrator_child_sup,ns_orchestrator_sup,

                  mb_master_sup,mb_master,leader_registry_sup,

                  leader_services_sup,<0.22005.0>,ns_server_sup,

                  ns_server_nodes_sup,<0.258.0>,ns_server_cluster_sup,

                  root_sup,<0.140.0>]

    message_queue_len: 0

    messages: []

    links: [<0.23486.0>]

    dictionary: []

    trap_exit: false

    status: running

    heap_size: 75113

    stack_size: 27

    reductions: 12440

  neighbours:

[user:error,2021-08-26T17:00:35.562-07:00,ns_1@172.23.97.74:<0.23486.0>:ns_orchestrator:log_rebalance_completion:1416]Rebalance exited with reason {buckets_cleanup_failed,['ns_1@172.23.123.26']}.

Rebalance Operation Id = b06d4c675802ff97cec996a5bcad0a01

[rebalance:error,2021-08-26T17:04:19.613-07:00,ns_1@172.23.97.74:<0.11938.1854>:ns_rebalancer:maybe_cleanup_old_buckets:941]Failed to cleanup old buckets on node 'ns_1@172.23.123.33': {badrpc,

                                                             {'EXIT',timeout}}

[rebalance:error,2021-08-26T17:04:19.613-07:00,ns_1@172.23.97.74:<0.11938.1854>:ns_rebalancer:maybe_cleanup_old_buckets:941]Failed to cleanup old buckets on node 'ns_1@172.23.120.77': {badrpc,

                                                             {'EXIT',timeout}}

[rebalance:error,2021-08-26T17:04:19.613-07:00,ns_1@172.23.97.74:<0.11938.1854>:ns_rebalancer:maybe_cleanup_old_buckets:941]Failed to cleanup old buckets on node 'ns_1@172.23.120.86': {badrpc,

                                                             {'EXIT',timeout}}

[ns_server:info,2021-08-26T17:04:19.614-07:00,ns_1@172.23.97.74:rebalance_agent<0.22070.0>:rebalance_agent:handle_down:290]Rebalancer process <0.11938.1854> died (reason {buckets_cleanup_failed,

                                                ['ns_1@172.23.123.33',

                                                 'ns_1@172.23.120.77',

                                                 'ns_1@172.23.120.86']}).

[ns_server:debug,2021-08-26T17:04:19.614-07:00,ns_1@172.23.97.74:leader_activities<0.23455.0>:leader_activities:handle_activity_down:505]Activity terminated with reason {shutdown,

                                 {async_died,

                                  {raised,

                                   {exit,

                                    {buckets_cleanup_failed,

                                     ['ns_1@172.23.123.33',

                                      'ns_1@172.23.120.77',

                                      'ns_1@172.23.120.86']},

                                    [{ns_rebalancer,rebalance_kv,4,

                                      [{file,"src/ns_rebalancer.erl"},

                                       {line,602}]},

                                     {ns_rebalancer,rebalance_body,5,

                                      [{file,"src/ns_rebalancer.erl"},

                                       {line,556}]},

                                     {async,'-async_init/4-fun-1-',3,

                                      [{file,"src/async.erl"},

                                       {line,191}]}]}}}}. Activity:

{activity,<0.7257.1854>,#Ref<0.3648888551.3028811779.97808>,default,

          <<"b0b0e2db350ee191d257c48381aff17e">>,

          [rebalance],

          majority,[]}

[error_logger:error,2021-08-26T17:04:19.616-07:00,ns_1@172.23.97.74:<0.19021.1854>:ale_error_logger_handler:do_log:101]

=========================CRASH REPORT=========================

  crasher:

    initial call: erlang:apply/2

    pid: <0.19021.1854>

    registered_name: []

    exception exit: {buckets_cleanup_failed,

                        ['ns_1@172.23.123.33','ns_1@172.23.120.77',

                         'ns_1@172.23.120.86']}

      in function  ns_rebalancer:rebalance_kv/4 (src/ns_rebalancer.erl, line 602)

      in call from ns_rebalancer:rebalance_body/5 (src/ns_rebalancer.erl, line 556)

      in call from async:'-async_init/4-fun-1-'/3 (src/async.erl, line 191)

    ancestors: [<0.23486.0>,ns_orchestrator_child_sup,ns_orchestrator_sup,

                  mb_master_sup,mb_master,leader_registry_sup,

                  leader_services_sup,<0.22005.0>,ns_server_sup,

                  ns_server_nodes_sup,<0.258.0>,ns_server_cluster_sup,

                  root_sup,<0.140.0>]

    message_queue_len: 0

    messages: []

    links: [<0.23486.0>]

    dictionary: []

    trap_exit: false

    status: running

    heap_size: 75113

    stack_size: 27

    reductions: 12440

  neighbours:

[user:error,2021-08-26T17:04:19.616-07:00,ns_1@172.23.97.74:<0.23486.0>:ns_orchestrator:log_rebalance_completion:1416]Rebalance exited with reason {buckets_cleanup_failed,

                                 ['ns_1@172.23.123.33','ns_1@172.23.120.77',

                                  'ns_1@172.23.120.86']}.

Rebalance Operation Id = 7fc214ef6dfb501e6e6fa4f73923631e