Swap rebalancing CBAS and KV node together is failing

2024-02-03 23:18:36,772 | test  | ERROR   | pool-14-thread-12 | [rest_client:_rebalance_status_and_progress:1470] {u'errorMessage': u'Rebalance failed. See logs for detailed reason. You can try again.', u'type': u'rebalance', u'masterRequestTimedOut': False, u'statusId': u'2a066d9dde32d0bbec7b02c23baece37', u'subtype': u'rebalance', u'statusIsStale': False, u'lastReportURI': u'/logs/rebalanceReport?reportID=335af95d6f98c6b7221987cdfbed1c34', u'status': u'notRunning'} - rebalance failed

2024-02-03 23:18:38,061 | test  | ERROR   | pool-14-thread-12 | [rest_client:print_UI_logs:2668] {u'code': 0, u'module': u'ns_orchestrator', u'type': u'critical', u'node': u'ns_1@ec2-54-221-19-50.compute-1.amazonaws.com', u'tstamp': 1707031108576L, u'shortText': u'message', u'serverTime': u'2024-02-03T23:18:28.576Z', u'text': u'Rebalance exited with reason {mover_crashed,\n                              {unexpected_exit,\n                               {\'EXIT\',<0.13194.11>,\n                                {{bulk_set_vbucket_state_failed,\n                                  [{\'ns_1@ec2-54-227-92-125.compute-1.amazonaws.com\',\n                                    {\'EXIT\',\n                                     {{{{{badmatch,\n                                          [{<37278.4618.7>,\n                                            {done,exit,\n                                             {{{badmatch,{error,etimedout}},\n                                               [{dcp_proxy,connect_inner,3,\n                                                 [{file,"src/dcp_proxy.erl"},\n                                                  {line,299}]},\n                                                {dcp_proxy,connect,5,\n                                                 [{file,"src/dcp_proxy.erl"},\n                                                  {line,252}]},\n                                                {dcp_proxy,maybe_connect,2,\n                                                 [{file,"src/dcp_proxy.erl"},\n                                                  {line,235}]},\n                                                {dcp_producer_conn,\n                                                 handle_call,4,\n                                                 [{file,\n                                                   "src/dcp_producer_conn.erl"},\n                                                  {line,50}]},\n                                                {dcp_proxy,handle_call,3,\n                                                 [{file,"src/dcp_proxy.erl"},\n                                                  {line,154}]},\n                                                {gen_server,try_handle_call,\n                                                 4,\n                                                 [{file,"gen_server.erl"},\n                                                  {line,1149}]},\n                                                {gen_server,handle_msg,6,\n                                                 [{file,"gen_server.erl"},\n                                                  {line,1178}]},\n                                                {proc_lib,init_p_do_apply,3,\n                                                 [{file,"proc_lib.erl"},\n                                                  {line,240}]}]},\n                                              {gen_server,call,\n                                               [<37278.4617.7>,\n                                                {connect,\n                                                 [collections,del_times,\n                                                  del_user_xattr,json,\n                                                  set_consumer_name,snappy,\n                                                  ssl,xattr]},\n                                                infinity]}},\n                                             [{gen_server,call,3,\n                                               [{file,"gen_server.erl"},\n                                                {line,385}]},\n                                              {dcp_replicator,\n                                               connect_to_producer,3,\n                                               [{file,\n                                                 "src/dcp_replicator.erl"},\n                                                {line,76}]},\n                                              {dcp_replicator,\n                                               \'-spawn_and_wait/1-fun-0-\',1,\n                                               [{file,\n                                                 "src/dcp_replicator.erl"},\n                                                {line,323}]}]}}]},\n                                         [{misc,\n                                           sync_shutdown_many_i_am_trapping_exits,\n                                           1,\n                                           [{file,"src/misc.erl"},\n                                            {line,1517}]},\n                                          {dcp_replicator,spawn_and_wait,1,\n                                           [{file,"src/dcp_replicator.erl"},\n                                            {line,344}]},\n                                          {dcp_replicator,handle_info,2,\n                                           [{file,"src/dcp_replicator.erl"},\n                                            {line,137}]},\n                                          {gen_server,try_dispatch,4,\n                                           [{file,"gen_server.erl"},\n                                            {line,1123}]},\n                                          {gen_server,handle_msg,6,\n                                           [{file,"gen_server.erl"},\n                                            {line,1200}]},\n                                          {proc_lib,init_p_do_apply,3,\n                                           [{file,"proc_lib.erl"},\n                                            {line,240}]}]},\n                                        {gen_server,call,\n                                         [<37278.4567.7>,\n                                          {setup_replication,[1020]},\n                                          infinity]}},\n                                       {gen_server,call,\n                                        [\'replication_manager-default\',\n                                         {change_vbucket_replication,1020,\n                                          \'ns_1@ec2-54-160-132-252.compute-1.amazonaws.com\'},\n                                         infinity]}},\n                                      {gen_server,call,\n                                       [{\'janitor_agent-default\',\n                                         \'ns_1@ec2-54-227-92-125.compute-1.amazonaws.com\'},\n                                        {if_rebalance,<0.13149.11>,\n                                         {update_vbucket_state,1022,replica,\n                                          passive,\n                                          \'ns_1@ec2-54-160-132-252.compute-1.amazonaws.com\'}},\n                                        infinity]}}}}]},\n                                 [{janitor_agent,bulk_set_vbucket_state,4,\n                                   [{file,"src/janitor_agent.erl"},\n                                    {line,404}]},\n                                  {proc_lib,init_p,3,\n                                   [{file,"proc_lib.erl"},{line,225}]}]}}}}.\nRebalance Operation Id = 566df57fffb466fa81b9f67b305b7ff2'}

2024-02-03 23:18:38,062 | test  | ERROR   | pool-14-thread-12 | [rest_client:print_UI_logs:2668] {u'code': 0, u'module': u'ns_vbucket_mover', u'type': u'critical', u'node': u'ns_1@ec2-54-221-19-50.compute-1.amazonaws.com', u'tstamp': 1707031108559L, u'shortText': u'message', u'serverTime': u'2024-02-03T23:18:28.559Z', u'text': u'Worker <0.13161.11> (for action {move,{1022,\n                                       [\'ns_1@ec2-54-160-132-252.compute-1.amazonaws.com\'],\n                                       [\'ns_1@ec2-54-227-92-125.compute-1.amazonaws.com\'],\n                                       []}}) exited with reason {unexpected_exit,\n                                                                 {\'EXIT\',\n                                                                  <0.13194.11>,\n                                                                  {{bulk_set_vbucket_state_failed,\n                                                                    [{\'ns_1@ec2-54-227-92-125.compute-1.amazonaws.com\',\n                                                                      {\'EXIT\',\n                                                                       {{{{{badmatch,\n                                                                            [{<37278.4618.7>,\n                                                                              {done,\n                                                                               exit,\n                                                                               {{{badmatch,\n                                                                                  {error,\n                                                                                   etimedout}},\n                                                                                 [{dcp_proxy,\n                                                                                   connect_inner,\n                                                                                   3,\n                                                                                   [{file,\n                                                                                     "src/dcp_proxy.erl"},\n                                                                                    {line,\n                                                                                     299}]},\n                                                                                  {dcp_proxy,\n                                                                                   connect,\n                                                                                   5,\n                                                                                   [{file,\n                                                                                     "src/dcp_proxy.erl"},\n                                                                                    {line,\n                                                                                     252}]},\n                                                                                  {dcp_proxy,\n                                                                                   maybe_connect,\n                                                                                   2,\n                                                                                   [{file,\n                                                                                     "src/dcp_proxy.erl"},\n                                                                                    {line,\n                                                                                     235}]},\n                                                                                  {dcp_producer_conn,\n                                                                                   handle_call,\n                                                                                   4,\n                                                                                   [{file,\n                                                                                     "src/dcp_producer_conn.erl"},\n                                                                                    {line,\n                                                                                     50}]},\n                                                                                  {dcp_proxy,\n                                                                                   handle_call,\n                                                                                   3,\n                                                                                   [{file,\n                                                                                     "src/dcp_proxy.erl"},\n                                                                                    {line,\n                                                                                     154}]},\n                                                                                  {gen_server,\n                                                                                   try_handle_call,\n                                                                                   4,\n                                                                                   [{file,\n                                                                                     "gen_server.erl"},\n                                                                                    {line,\n                                                                                     1149}]},\n                                                                                  {gen_server,\n                                                                                   handle_msg,\n                                                                                   6,\n                                                                                   [{file,\n                                                                                     "gen_server.erl"},\n                                                                                    {line,\n                                                                                     1178}]},\n                                                                                  {proc_lib,\n                                                                                   init_p_do_apply,\n                                                                                   3,\n                                                                                   [{file,\n                                                                                     "proc_lib.erl"},\n                                                                                    {line,\n                                                                                     240}]}]},\n                                                                                {gen_server,\n                                                                                 call,\n                                                                                 [<37278.4617.7>,\n                                                                                  {connect,\n                                                                                   [collections,\n                                                                                    del_times,\n                                                                                    del_user_xattr,\n                                                                                    json,\n                                                                                    set_consumer_name,\n                                                                                    snappy,\n                                                                                    ssl,\n                                                                                    xattr]},\n                                                                                  infinity]}},\n                                                                               [{gen_server,\n                                                                                 call,\n                                                                                 3,\n                                                                                 [{file,\n                                                                                   "gen_server.erl"},\n                                                                                  {line,\n                                                                                   385}]},\n                                                                                {dcp_replicator,\n                                                                                 connect_to_producer,\n                                                                                 3,\n                                                                                 [{file,\n                                                                                   "src/dcp_replicator.erl"},\n                                                                                  {line,\n                                                                                   76}]},\n                                                                                {dcp_replicator,\n                                                                                 \'-spawn_and_wait/1-fun-0-\',\n                                                                                 1,\n                                                                                 [{file,\n                                                                                   "src/dcp_replicator.erl"},\n                                                                                  {line,\n                                                                                   323}]}]}}]},\n                                                                           [{misc,\n                                                                             sync_shutdown_many_i_am_trapping_exits,\n                                                                             1,\n                                                                             [{file,\n                                                                               "src/misc.erl"},\n                                                                              {line,\n                                                                               1517}]},\n                                                                            {dcp_replicator,\n                                                                             spawn_and_wait,\n                                                                             1,\n                                                                             [{file,\n                                                                               "src/dcp_replicator.erl"},\n                                                                              {line,\n                                                                               344}]},\n                                                                            {dcp_replicator,\n                                                                             handle_info,\n                                                                             2,\n                                                                             [{file,\n                                                                               "src/dcp_replicator.erl"},\n                                                                              {line,\n                                                                               137}]},\n                                                                            {gen_server,\n                                                                             try_dispatch,\n                                                                             4,\n                                                                             [{file,\n                                                                               "gen_server.erl"},\n                                                                              {line,\n                                                                               1123}]},\n                                                                            {gen_server,\n                                                                             handle_msg,\n                                                                             6,\n                                                                             [{file,\n                                                                               "gen_server.erl"},\n                                                                              {line,\n                                                                               1200}]},\n                                                                            {proc_lib,\n                                                                             init_p_do_apply,\n                                                                             3,\n                                                                             [{file,\n                                                                               "proc_lib.erl"},\n                                                                              {line,\n                                                                               240}]}]},\n                                                                          {gen_server,\n                                                                           call,\n                                                                           [<37278.4567.7>,\n                                                                            {setup_replication,\n                                                                             [1020]},\n                                                                            infinity]}},\n                                                                         {gen_server,\n                                                                          call,\n                                                                          [\'replication_manager-default\',\n                                                                           {change_vbucket_replication,\n                                                                            1020,\n                                                                            \'ns_1@ec2-54-160-132-252.compute-1.amazonaws.com\'},\n                                                                           infinity]}},\n                                                                        {gen_server,\n                                                                         call,\n                                                                         [{\'janitor_agent-default\',\n                                                                           \'ns_1@ec2-54-227-92-125.compute-1.amazonaws.com\'},\n                                                                          {if_rebalance,\n                                                                           <0.13149.11>,\n                                                                           {update_vbucket_state,\n                                                                            1022,\n                                                                            replica,\n                                                                            passive,\n                                                                            \'ns_1@ec2-54-160-132-252.compute-1.amazonaws.com\'}},\n                                                                          infinity]}}}}]},\n                                                                   [{janitor_agent,\n                                                                     bulk_set_vbucket_state,\n                                                                     4,\n                                                                     [{file,\n                                                                       "src/janitor_agent.erl"},\n                                                                      {line,\n                                                                       404}]},\n                                                                    {proc_lib,\n                                                                     init_p,3,\n                                                                     [{file,\n                                                                       "proc_lib.erl"},\n                                                                      {line,\n                                                                       225}]}]}}}'}