CouchStore: 1-Scope, 100 Collections: ThrowExceptionUnderflowPolicy current:0 arg:-422

{u'code': 0, u'module': u'ns_orchestrator', u'type': u'critical', u'node': u'ns_1@172.23.120.170', u'tstamp': 1612511097424L, u'shortText': u'message', u'serverTime': u'2021-02-04T23:44:57.424Z', u'text': u'Rebalance exited with reason {mover_crashed,\n                              {unexpected_exit,\n                               {\'EXIT\',<0.21356.4>,\n                                {{bulk_set_vbucket_state_failed,\n                                  [{\'ns_1@172.23.121.129\',\n                                    {\'EXIT\',\n                                     {{{{{case_clause,\n                                          {error,\n                                           {{{badmatch,\n                                              {error,\n                                               {{badmatch,\n                                                 {error,econnrefused}},\n                                                [{dcp_proxy,connect,5,\n                                                  [{file,"src/dcp_proxy.erl"},\n                                                   {line,259}]},\n                                                 {dcp_proxy,maybe_connect,2,\n                                                  [{file,"src/dcp_proxy.erl"},\n                                                   {line,242}]},\n                                                 {dcp_producer_conn,init,2,\n                                                  [{file,\n                                                    "src/dcp_producer_conn.erl"},\n                                                   {line,31}]},\n                                                 {dcp_proxy,init,1,\n                                                  [{file,"src/dcp_proxy.erl"},\n                                                   {line,58}]},\n                                                 {gen_server,init_it,2,\n                                                  [{file,"gen_server.erl"},\n                                                   {line,374}]},\n                                                 {gen_server,init_it,6,\n                                                  [{file,"gen_server.erl"},\n                                                   {line,342}]},\n                                                 {proc_lib,init_p_do_apply,3,\n                                                  [{file,"proc_lib.erl"},\n                                                   {line,249}]}]}}},\n                                             [{dcp_replicator,init,1,\n                                               [{file,\n                                                 "src/dcp_replicator.erl"},\n                                                {line,50}]},\n                                              {gen_server,init_it,2,\n                                               [{file,"gen_server.erl"},\n                                                {line,374}]},\n                                              {gen_server,init_it,6,\n                                               [{file,"gen_server.erl"},\n                                                {line,342}]},\n                                              {proc_lib,init_p_do_apply,3,\n                                               [{file,"proc_lib.erl"},\n                                                {line,249}]}]},\n                                            {child,undefined,\n                                             {\'ns_1@172.23.121.123\',\n                                              [collections,del_times,\n                                               del_user_xattr,json,\n                                               set_consumer_name,snappy,\n                                               xattr]},\n                                             {dcp_replicator,start_link,\n                                              [\'ns_1@172.23.121.123\',\n                                               "GleamBook",\n                                               [collections,del_times,\n                                                del_user_xattr,json,\n                                                set_consumer_name,snappy,\n                                                xattr]]},\n                                             temporary,60000,worker,\n                                             [dcp_replicator]}}}},\n                                         [{dcp_sup,start_replicator,2,\n                                           [{file,"src/dcp_sup.erl"},\n                                            {line,57}]},\n                                          {dcp_sup,\n                                           \'-manage_replicators/2-lc$^3/1-3-\',\n                                           2,\n                                           [{file,"src/dcp_sup.erl"},\n                                            {line,107}]},\n                                          {dcp_replication_manager,\n                                           handle_call,3,\n                                           [{file,\n                                             "src/dcp_replication_manager.erl"},\n                                            {line,89}]},\n                                          {gen_server,try_handle_call,4,\n                                           [{file,"gen_server.erl"},\n                                            {line,661}]},\n                                          {gen_server,handle_msg,6,\n                                           [{file,"gen_server.erl"},\n                                            {line,690}]},\n                                          {proc_lib,init_p_do_apply,3,\n                                           [{file,"proc_lib.erl"},\n                                            {line,249}]}]},\n                                        {gen_server,call,\n                                         [\'dcp_replication_manager-GleamBook\',\n                                          {manage_replicators,\n                                           [\'ns_1@172.23.120.170\',\n                                            \'ns_1@172.23.121.115\',\n                                            \'ns_1@172.23.121.116\',\n                                            \'ns_1@172.23.121.123\',\n                                            \'ns_1@172.23.121.126\',\n                                            \'ns_1@172.23.121.127\',\n                                            \'ns_1@172.23.121.128\',\n                                            \'ns_1@172.23.121.131\',\n                                            \'ns_1@172.23.121.134\',\n                                            \'ns_1@172.23.121.135\']},\n                                          infinity]}},\n                                       {gen_server,call,\n                                        [\'replication_manager-GleamBook\',\n                                         {change_vbucket_replication,252,\n                                          \'ns_1@172.23.121.123\'},\n                                         infinity]}},\n                                      {gen_server,call,\n                                       [{\'janitor_agent-GleamBook\',\n                                         \'ns_1@172.23.121.129\'},\n                                        {if_rebalance,<0.20971.4>,\n                                         {update_vbucket_state,456,pending,\n                                          passive,\'ns_1@172.23.121.127\'}},\n                                        infinity]}}}}]},\n                                 [{janitor_agent,bulk_set_vbucket_state,4,\n                                   [{file,"src/janitor_agent.erl"},\n                                    {line,403}]},\n                                  {proc_lib,init_p,3,\n                                   [{file,"proc_lib.erl"},{line,234}]}]}}}}.\nRebalance Operation Id = ba79d1afef1ebcc2e69112dbe6513c68'}

2021-02-04 23:45:05,716 | test  | ERROR   | pool-1-thread-14 | [rest_client:print_UI_logs:2581] {u'code': 0, u'module': u'ns_vbucket_mover', u'type': u'critical', u'node': u'ns_1@172.23.120.170', u'tstamp': 1612511097310L, u'shortText': u'message', u'serverTime': u'2021-02-04T23:44:57.310Z', u'text': u'Worker <0.21099.4> (for action {move,{456,\n                                      [\'ns_1@172.23.121.127\',\n                                       \'ns_1@172.23.121.129\',\n                                       \'ns_1@172.23.121.128\'],\n                                      [\'ns_1@172.23.121.129\',\n                                       \'ns_1@172.23.121.128\',\n                                       \'ns_1@172.23.121.127\'],\n                                      []}}) exited with reason {unexpected_exit,\n                                                                {\'EXIT\',\n                                                                 <0.21356.4>,\n                                                                 {{bulk_set_vbucket_state_failed,\n                                                                   [{\'ns_1@172.23.121.129\',\n                                                                     {\'EXIT\',\n                                                                      {{{{{case_clause,\n                                                                           {error,\n                                                                            {{{badmatch,\n                                                                               {error,\n                                                                                {{badmatch,\n                                                                                  {error,\n                                                                                   econnrefused}},\n                                                                                 [{dcp_proxy,\n                                                                                   connect,\n                                                                                   5,\n                                                                                   [{file,\n                                                                                     "src/dcp_proxy.erl"},\n                                                                                    {line,\n                                                                                     259}]},\n                                                                                  {dcp_proxy,\n                                                                                   maybe_connect,\n                                                                                   2,\n                                                                                   [{file,\n                                                                                     "src/dcp_proxy.erl"},\n                                                                                    {line,\n                                                                                     242}]},\n                                                                                  {dcp_producer_conn,\n                                                                                   init,\n                                                                                   2,\n                                                                                   [{file,\n                                                                                     "src/dcp_producer_conn.erl"},\n                                                                                    {line,\n                                                                                     31}]},\n                                                                                  {dcp_proxy,\n                                                                                   init,\n                                                                                   1,\n                                                                                   [{file,\n                                                                                     "src/dcp_proxy.erl"},\n                                                                                    {line,\n                                                                                     58}]},\n                                                                                  {gen_server,\n                                                                                   init_it,\n                                                                                   2,\n                                                                                   [{file,\n                                                                                     "gen_server.erl"},\n                                                                                    {line,\n                                                                                     374}]},\n                                                                                  {gen_server,\n                                                                                   init_it,\n                                                                                   6,\n                                                                                   [{file,\n                                                                                     "gen_server.erl"},\n                                                                                    {line,\n                                                                                     342}]},\n                                                                                  {proc_lib,\n                                                                                   init_p_do_apply,\n                                                                                   3,\n                                                                                   [{file,\n                                                                                     "proc_lib.erl"},\n                                                                                    {line,\n                                                                                     249}]}]}}},\n                                                                              [{dcp_replicator,\n                                                                                init,\n                                                                                1,\n                                                                                [{file,\n                                                                                  "src/dcp_replicator.erl"},\n                                                                                 {line,\n                                                                                  50}]},\n                                                                               {gen_server,\n                                                                                init_it,\n                                                                                2,\n                                                                                [{file,\n                                                                                  "gen_server.erl"},\n                                                                                 {line,\n                                                                                  374}]},\n                                                                               {gen_server,\n                                                                                init_it,\n                                                                                6,\n                                                                                [{file,\n                                                                                  "gen_server.erl"},\n                                                                                 {line,\n                                                                                  342}]},\n                                                                               {proc_lib,\n                                                                                init_p_do_apply,\n                                                                                3,\n                                                                                [{file,\n                                                                                  "proc_lib.erl"},\n                                                                                 {line,\n                                                                                  249}]}]},\n                                                                             {child,\n                                                                              undefined,\n                                                                              {\'ns_1@172.23.121.123\',\n                                                                               [collections,\n                                                                                del_times,\n                                                                                del_user_xattr,\n                                                                                json,\n                                                                                set_consumer_name,\n                                                                                snappy,\n                                                                                xattr]},\n                                                                              {dcp_replicator,\n                                                                               start_link,\n                                                                               [\'ns_1@172.23.121.123\',\n                                                                                "GleamBook",\n                                                                                [collections,\n                                                                                 del_times,\n                                                                                 del_user_xattr,\n                                                                                 json,\n                                                                                 set_consumer_name,\n                                                                                 snappy,\n                                                                                 xattr]]},\n                                                                              temporary,\n                                                                              60000,\n                                                                              worker,\n                                                                              [dcp_replicator]}}}},\n                                                                          [{dcp_sup,\n                                                                            start_replicator,\n                                                                            2,\n                                                                            [{file,\n                                                                              "src/dcp_sup.erl"},\n                                                                             {line,\n                                                                              57}]},\n                                                                           {dcp_sup,\n                                                                            \'-manage_replicators/2-lc$^3/1-3-\',\n                                                                            2,\n                                                                            [{file,\n                                                                              "src/dcp_sup.erl"},\n                                                                             {line,\n                                                                              107}]},\n                                                                           {dcp_replication_manager,\n                                                                            handle_call,\n                                                                            3,\n                                                                            [{file,\n                                                                              "src/dcp_replication_manager.erl"},\n                                                                             {line,\n                                                                              89}]},\n                                                                           {gen_server,\n                                                                            try_handle_call,\n                                                                            4,\n                                                                            [{file,\n                                                                              "gen_server.erl"},\n                                                                             {line,\n                                                                              661}]},\n                                                                           {gen_server,\n                                                                            handle_msg,\n                                                                            6,\n                                                                            [{file,\n                                                                              "gen_server.erl"},\n                                                                             {line,\n                                                                              690}]},\n                                                                           {proc_lib,\n                                                                            init_p_do_apply,\n                                                                            3,\n                                                                            [{file,\n                                                                              "proc_lib.erl"},\n                                                                             {line,\n                                                                              249}]}]},\n                                                                         {gen_server,\n                                                                          call,\n                                                                          [\'dcp_replication_manager-GleamBook\',\n                                                                           {manage_replicators,\n                                                                            [\'ns_1@172.23.120.170\',\n                                                                             \'ns_1@172.23.121.115\',\n                                                                             \'ns_1@172.23.121.116\',\n                                                                             \'ns_1@172.23.121.123\',\n                                                                             \'ns_1@172.23.121.126\',\n                                                                             \'ns_1@172.23.121.127\',\n                                                                             \'ns_1@172.23.121.128\',\n                                                                             \'ns_1@172.23.121.131\',\n                                                                             \'ns_1@172.23.121.134\',\n                                                                             \'ns_1@172.23.121.135\']},\n                                                                           infinity]}},\n                                                                        {gen_server,\n                                                                         call,\n                                                                         [\'replication_manager-GleamBook\',\n                                                                          {change_vbucket_replication,\n                                                                           252,\n                                                                           \'ns_1@172.23.121.123\'},\n                                                                          infinity]}},\n                                                                       {gen_server,\n                                                                        call,\n                                                                        [{\'janitor_agent-GleamBook\',\n                                                                          \'ns_1@172.23.121.129\'},\n                                                                         {if_rebalance,\n                                                                          <0.20971.4>,\n                                                                          {update_vbucket_state,\n                                                                           456,\n                                                                           pending,\n                                                                           passive,\n                                                                           \'ns_1@172.23.121.127\'}},\n                                                                         infinity]}}}}]},\n                                                                  [{janitor_agent,\n                                                                    bulk_set_vbucket_state,\n                                                                    4,\n                                                                    [{file,\n                                                                      "src/janitor_agent.erl"},\n                                                                     {line,\n                                                                      403}]},\n                                                                   {proc_lib,\n                                                                    init_p,3,\n                                                                    [{file,\n                                                                      "proc_lib.erl"},\n                                                                     {line,\n                                                                      234}]}]}}}'}