[System Test]: Rebalance failed with mover crashed due to failed_to_update_vbucket_map

[2020-12-04T14:57:57-08:00, sequoiatools/couchbase-cli:6.5:17f5f3] server-add -c 172.23.104.16:8091 --server-add https://172.23.104.17 -u Administrator -p password --server-add-username Administrator --server-add-password password --services data

[2020-12-04T14:58:20-08:00, sequoiatools/couchbase-cli:6.5:91b454] server-add -c 172.23.104.16:8091 --server-add https://172.23.104.18 -u Administrator -p password --server-add-username Administrator --server-add-password password --services data

[2020-12-04T14:58:40-08:00, sequoiatools/couchbase-cli:6.5:428922] server-add -c 172.23.104.16:8091 --server-add https://172.23.104.23 -u Administrator -p password --server-add-username Administrator --server-add-password password --services eventing

[2020-12-04T14:58:50-08:00, sequoiatools/couchbase-cli:6.5:32045c] rebalance -c 172.23.104.16:8091 -u Administrator -p password 

[ns_server:error,2020-12-04T15:27:32.144-08:00,ns_1@172.23.104.16:<0.18610.1018>:ns_single_vbucket_mover:spawn_and_wait:82]Got unexpected exit signal {'EXIT',<0.2395.1015>,

                            {unexpected_exit,

                             {'EXIT',<0.15328.1018>,

                              {failed_to_update_vbucket_map,"bucket_op",743,

                               {error,

                                [{'ns_1@172.23.96.31',timeout},

                                 {'ns_1@172.23.97.82',timeout},

                                 {'ns_1@172.23.97.83',timeout}]}}}}}

[ns_server:error,2020-12-04T15:27:32.144-08:00,ns_1@172.23.104.16:<0.18610.1018>:misc:sync_shutdown_many_i_am_trapping_exits:1372]Shutdown of the following failed: [{<0.17760.1016>,normal}]

[ns_server:error,2020-12-04T15:27:32.144-08:00,ns_1@172.23.104.16:<0.18610.1018>:misc:try_with_maybe_ignorant_after:1408]Eating exception from ignorant after-block:

{error,{badmatch,[{<0.17760.1016>,normal}]},

       [{misc,sync_shutdown_many_i_am_trapping_exits,1,

              [{file,"src/misc.erl"},{line,1374}]},

        {misc,try_with_maybe_ignorant_after,2,

              [{file,"src/misc.erl"},{line,1406}]},

        {ns_single_vbucket_mover,mover,6,

                                 [{file,"src/ns_single_vbucket_mover.erl"},

                                  {line,52}]},

        {proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,247}]}]}

[user:error,2020-12-04T15:27:32.155-08:00,ns_1@172.23.104.16:<0.7127.0>:ns_orchestrator:log_rebalance_completion:1449]Rebalance exited with reason {mover_crashed,

                              {unexpected_exit,

                               {'EXIT',<0.15328.1018>,

                                {failed_to_update_vbucket_map,"bucket_op",

                                 743,

                                 {error,

                                  [{'ns_1@172.23.96.31',timeout},

                                   {'ns_1@172.23.97.82',timeout},

                                   {'ns_1@172.23.97.83',timeout}]}}}}}.

Rebalance Operation Id = 932f1680cd9c7f1cfc69fdafa3672381

[ns_server:error,2020-12-04T15:27:33.923-08:00,ns_1@172.23.104.16:service_status_keeper_worker<0.5656.0>:service_status_keeper:grab_status:222]Got exception while getting items from node 'ns_1@172.23.96.31': {exit,

                                                                  {timeout,

                                                                   {gen_server,

                                                                    call,

                                                                    [{remote_api,

                                                                      'ns_1@172.23.96.31'},

                                                                     {get_service_remote_items,

                                                                      service_eventing},

                                                                     10000]}}}