2020-11-19 21:04:39,868 | test | ERROR | pool-1-thread-15 | [rest_client:_rebalance_status_and_progress:1479] {u'errorMessage': u'Rebalance failed. See logs for detailed reason. You can try again.', u'status': u'none'} - rebalance failed 2020-11-19 21:04:40,006 | test | INFO | pool-1-thread-15 | [rest_client:print_UI_logs:2583] Latest logs from UI on 172.23.120.137: 2020-11-19 21:04:40,006 | test | ERROR | pool-1-thread-15 | [rest_client:print_UI_logs:2585] {u'code': 0, u'module': u'ns_orchestrator', u'type': u'critical', u'node': u'ns_1@172.23.120.137', u'tstamp': 1605848678249L, u'shortText': u'message', u'serverTime': u'2020-11-19T21:04:38.249Z', u'text': u'Rebalance exited with reason {mover_crashed,\n {unexpected_exit,\n {\'EXIT\',<0.26892.2>,\n {{{{{child_interrupted,\n {\'EXIT\',<20133.4477.0>,socket_closed}},\n [{dcp_replicator,spawn_and_wait,1,\n [{file,"src/dcp_replicator.erl"},\n {line,265}]},\n {dcp_replicator,handle_call,3,\n [{file,"src/dcp_replicator.erl"},\n {line,121}]},\n {gen_server,try_handle_call,4,\n [{file,"gen_server.erl"},{line,661}]},\n {gen_server,handle_msg,6,\n [{file,"gen_server.erl"},{line,690}]},\n {proc_lib,init_p_do_apply,3,\n [{file,"proc_lib.erl"},{line,249}]}]},\n {gen_server,call,\n [<20133.4476.0>,\n {setup_replication,\n [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,\n 16,17,18,19,20,21,22,23,24,25,26,27,28,\n 29,30,31,32,33,34,35,36,37,38,39,40,41,\n 42,43,44,45,46,47,48,49,50,51,52,53,54,\n 55,56,57,58,59,60,61,62,63,64,65,66,67,\n 68,69,70,71,72,73,74,75,76,77,78,79,80,\n 81,82,83,84,85,86,87,88,89,90,91,92,93,\n 94,95,96,97,98,99,100,101,102,103,104,\n 105,106,107,108,109,110,111,112,113,\n 114,115,116,117,118,119,120,121,122,\n 123,124,125,126,127,128,129,130,131,\n 132,133,134,135,136,137,138,139,140,\n 141,142,143,144,145,146,147,148,149,\n 150,151,152,153,154,155,156,157,158,\n 159,160,161,162,163,164,165,166,167,\n 168,169,170,171,172,173,174,175,176,\n 177,178,179,180,181,182,183,184,185,\n 186,187,188,189,190,191,192,193,194,\n 195,196,197,198,199,200,201,202]},\n infinity]}},\n {gen_server,call,\n [\'replication_manager-default\',\n {change_vbucket_replication,203,undefined},\n infinity]}},\n {gen_server,call,\n [{\'janitor_agent-default\',\n \'ns_1@172.23.120.140\'},\n {if_rebalance,<0.4495.2>,\n {update_vbucket_state,971,active,paused,\n undefined,\n [[\'ns_1@172.23.120.140\',\n \'ns_1@172.23.120.139\',\n \'ns_1@172.23.120.138\',\n \'ns_1@172.23.120.137\']]}},\n infinity]}}}}}.\nRebalance Operation Id = 461e6013be113bf13dfa72bcd5423ee8'} 2020-11-19 21:04:40,009 | test | ERROR | pool-1-thread-15 | [rest_client:print_UI_logs:2585] {u'code': 0, u'module': u'ns_vbucket_mover', u'type': u'critical', u'node': u'ns_1@172.23.120.137', u'tstamp': 1605848677897L, u'shortText': u'message', u'serverTime': u'2020-11-19T21:04:37.897Z', u'text': u'Worker <0.26582.2> (for action {move,{971,\n [\'ns_1@172.23.120.140\',\n \'ns_1@172.23.120.139\',\n \'ns_1@172.23.120.138\',\n \'ns_1@172.23.120.137\'],\n [\'ns_1@172.23.120.142\',\n \'ns_1@172.23.120.147\',\n \'ns_1@172.23.120.138\',\n \'ns_1@172.23.120.137\'],\n []}}) exited with reason {unexpected_exit,\n {\'EXIT\',\n <0.26892.2>,\n {{{{{child_interrupted,\n {\'EXIT\',\n <20133.4477.0>,\n socket_closed}},\n [{dcp_replicator,\n spawn_and_wait,\n 1,\n [{file,\n "src/dcp_replicator.erl"},\n {line,\n 265}]},\n {dcp_replicator,\n handle_call,\n 3,\n [{file,\n "src/dcp_replicator.erl"},\n {line,\n 121}]},\n {gen_server,\n try_handle_call,\n 4,\n [{file,\n "gen_server.erl"},\n {line,\n 661}]},\n {gen_server,\n handle_msg,\n 6,\n [{file,\n "gen_server.erl"},\n {line,\n 690}]},\n {proc_lib,\n init_p_do_apply,\n 3,\n [{file,\n "proc_lib.erl"},\n {line,\n 249}]}]},\n {gen_server,\n call,\n [<20133.4476.0>,\n {setup_replication,\n [0,1,2,\n 3,4,5,\n 6,7,8,\n 9,10,\n 11,12,\n 13,14,\n 15,16,\n 17,18,\n 19,20,\n 21,22,\n 23,24,\n 25,26,\n 27,28,\n 29,30,\n 31,32,\n 33,34,\n 35,36,\n 37,38,\n 39,40,\n 41,42,\n 43,44,\n 45,46,\n 47,48,\n 49,50,\n 51,52,\n 53,54,\n 55,56,\n 57,58,\n 59,60,\n 61,62,\n 63,64,\n 65,66,\n 67,68,\n 69,70,\n 71,72,\n 73,74,\n 75,76,\n 77,78,\n 79,80,\n 81,82,\n 83,84,\n 85,86,\n 87,88,\n 89,90,\n 91,92,\n 93,94,\n 95,96,\n 97,98,\n 99,\n 100,\n 101,\n 102,\n 103,\n 104,\n 105,\n 106,\n 107,\n 108,\n 109,\n 110,\n 111,\n 112,\n 113,\n 114,\n 115,\n 116,\n 117,\n 118,\n 119,\n 120,\n 121,\n 122,\n 123,\n 124,\n 125,\n 126,\n 127,\n 128,\n 129,\n 130,\n 131,\n 132,\n 133,\n 134,\n 135,\n 136,\n 137,\n 138,\n 139,\n 140,\n 141,\n 142,\n 143,\n 144,\n 145,\n 146,\n 147,\n 148,\n 149,\n 150,\n 151,\n 152,\n 153,\n 154,\n 155,\n 156,\n 157,\n 158,\n 159,\n 160,\n 161,\n 162,\n 163,\n 164,\n 165,\n 166,\n 167,\n 168,\n 169,\n 170,\n 171,\n 172,\n 173,\n 174,\n 175,\n 176,\n 177,\n 178,\n 179,\n 180,\n 181,\n 182,\n 183,\n 184,\n 185,\n 186,\n 187,\n 188,\n 189,\n 190,\n 191,\n 192,\n 193,\n 194,\n 195,\n 196,\n 197,\n 198,\n 199,\n 200,\n 201,\n 202]},\n infinity]}},\n {gen_server,\n call,\n [\'replication_manager-default\',\n {change_vbucket_replication,\n 203,\n undefined},\n infinity]}},\n {gen_server,\n call,\n [{\'janitor_agent-default\',\n \'ns_1@172.23.120.140\'},\n {if_rebalance,\n <0.4495.2>,\n {update_vbucket_state,\n 971,\n active,\n paused,\n undefined,\n [[\'ns_1@172.23.120.140\',\n \'ns_1@172.23.120.139\',\n \'ns_1@172.23.120.138\',\n \'ns_1@172.23.120.137\']]}},\n infinity]}}}}'} 2020-11-19 21:04:40,015 | test | ERROR | pool-1-thread-15 | [rest_client:print_UI_logs:2585] {u'code': 0, u'module': u'ns_vbucket_mover', u'type': u'info', u'node': u'ns_1@172.23.120.137', u'tstamp': 1605848666499L, u'shortText': u'message', u'serverTime': u'2020-11-19T21:04:26.499Z', u'text': u'Bucket "default" rebalance appears to be swap rebalance'} 2020-11-19 21:04:40,016 | test | ERROR | pool-1-thread-15 | [rest_client:print_UI_logs:2585] {u'code': 0, u'module': u'ns_memcached', u'type': u'info', u'node': u'ns_1@172.23.120.147', u'tstamp': 1605848665263L, u'shortText': u'message', u'serverTime': u'2020-11-19T21:04:25.263Z', u'text': u'Bucket "default" loaded on node \'ns_1@172.23.120.147\' in 0 seconds.'} 2020-11-19 21:04:40,016 | test | ERROR | pool-1-thread-15 | [rest_client:print_UI_logs:2585] {u'code': 0, u'module': u'ns_memcached', u'type': u'info', u'node': u'ns_1@172.23.120.142', u'tstamp': 1605848665250L, u'shortText': u'message', u'serverTime': u'2020-11-19T21:04:25.250Z', u'text': u'Bucket "default" loaded on node \'ns_1@172.23.120.142\' in 0 seconds.'} 2020-11-19 21:04:40,016 | test | ERROR | pool-1-thread-15 | [rest_client:print_UI_logs:2585] {u'code': 0, u'module': u'ns_rebalancer', u'type': u'info', u'node': u'ns_1@172.23.120.137', u'tstamp': 1605848665051L, u'shortText': u'message', u'serverTime': u'2020-11-19T21:04:25.051Z', u'text': u'Started rebalancing bucket default'} 2020-11-19 21:04:40,019 | test | ERROR | pool-1-thread-15 | [rest_client:print_UI_logs:2585] {u'code': 0, u'module': u'ns_orchestrator', u'type': u'info', u'node': u'ns_1@172.23.120.137', u'tstamp': 1605848664616L, u'shortText': u'message', u'serverTime': u'2020-11-19T21:04:24.616Z', u'text': u"Starting rebalance, KeepNodes = ['ns_1@172.23.120.142','ns_1@172.23.120.138',\n 'ns_1@172.23.120.147','ns_1@172.23.120.137'], EjectNodes = ['ns_1@172.23.120.139',\n 'ns_1@172.23.120.140'], Failed over and being ejected nodes = []; no delta recovery nodes; Operation Id = 461e6013be113bf13dfa72bcd5423ee8"} 2020-11-19 21:04:40,020 | test | ERROR | pool-1-thread-15 | [rest_client:print_UI_logs:2585] {u'code': 0, u'module': u'memcached_config_mgr', u'type': u'info', u'node': u'ns_1@172.23.120.147', u'tstamp': 1605848664586L, u'shortText': u'message', u'serverTime': u'2020-11-19T21:04:24.586Z', u'text': u'Hot-reloaded memcached.json for config change of the following keys: [<<"scramsha_fallback_salt">>]'} 2020-11-19 21:04:40,022 | test | ERROR | pool-1-thread-15 | [rest_client:print_UI_logs:2585] {u'code': 3, u'module': u'ns_cluster', u'type': u'info', u'node': u'ns_1@172.23.120.147', u'tstamp': 1605848664531L, u'shortText': u'message', u'serverTime': u'2020-11-19T21:04:24.531Z', u'text': u'Node ns_1@172.23.120.147 joined cluster'} 2020-11-19 21:04:40,022 | test | ERROR | pool-1-thread-15 | [rest_client:print_UI_logs:2585] {u'code': 1, u'module': u'menelaus_sup', u'type': u'info', u'node': u'ns_1@172.23.120.147', u'tstamp': 1605848664430L, u'shortText': u'web start ok', u'serverTime': u'2020-11-19T21:04:24.430Z', u'text': u'Couchbase Server has started on web port 8091 on node \'ns_1@172.23.120.147\'. Version: "7.0.0-3800-enterprise".'}