Details
-
Bug
-
Resolution: Fixed
-
Test Blocker
-
4.5.0
-
None
-
Centos 6.5 64-bit
-
Untriaged
-
Centos 64-bit
-
Yes
Description
Run test
newupgradetests.MultiNodesUpgradeTests.online_upgrade_rebalance_in_out,initial_version=2.5.0-1059-rel,during-ops=start_index,ddocs-num=3,upgrade_version=4.5.0-2585,stop-on-failure=True,skip_cleanup=True,get-cbcollect-info=True,skip_init_check_cbserver=true
Start cluster with 2 nodes 2.5.0
Rebalance in 2 4.5.0-2585. Passed
Rebalance out 2 2.5.0 nodes. Failed
Error
Latest logs from UI on 172.23.106.57:
|
2016-05-19 15:47:03,569 - root - ERROR - {u'node': u'ns_1@172.23.107.143', u'code': 0, u'text': u'Bucket "default" loaded on node \'ns_1@172.23.107.143\' in 0 seconds.', u'shortText': u'message', u'serverTime': u'2016-05-19T15:46:53.787Z', u'module': u'ns_memcached', u'tstamp': 1463698013787, u'type': u'info'}
|
2016-05-19 15:47:03,569 - root - ERROR - {u'node': u'ns_1@172.23.105.108', u'code': 2, u'text': u'Rebalance exited with reason {badarg,\n [{erlang,\'++\',\n [{\'EXIT\',\n {{janitor_agent_servant_died,\n {{error,closed},\n {gen_server,call,\n [\'ns_memcached-default\',\n {get_mass_tap_docs_estimate,\n [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,\n 15,16,17,18,19,20,21,22,23,24,25,26,\n 27,28,29,30,31,32,33,34,35,36,37,38,\n 39,40,41,42,43,44,45,46,47,48,49,50,\n 51,52,53,54,55,56,57,58,59,60,61,62,\n 63,64,65,66,67,68,69,70,71,72,73,74,\n 75,76,77,78,79,80,81,82,83,84,85,86,\n 87,88,89,90,91,92,93,94,95,96,97,98,\n 99,100,101,102,103,104,105,106,107,\n 108,109,110,111,112,113,114,115,116,\n 117,118,119,120,121,122,123,124,125,\n 126,127,128,129,130,131,132,133,134,\n 135,136,137,138,139,140,141,142,143,\n 144,145,146,147,148,149,150,151,152,\n 153,154,155,156,157,158,159,160,161,\n 162,163,164,165,166,167,168,169,170,\n 171,172,173,174,175,176,177,178,179,\n 180,181,182,183,184,185,186,187,188,\n 189,190,191,192,193,194,195,196,197,\n 198,199,200,201,202,203,204,205,206,\n 207,208,209,210,211,212,213,214,215,\n 216,217,218,219,220,221,222,223,224,\n 225,226,227,228,229,230,231,232,233,\n 234,235,236,237,238,239,240,241,242,\n 243,244,245,246,247,248,249,250,251,\n 252,253,254,255,341,342,343,344,345,\n 346,347,348,349,350,351,352,353,354,\n 355,356,357,358,359,360,361,362,363,\n 364,365,366,367,368,369,370,371,372,\n 373,374,375,376,377,378,379,380,381,\n 382,383,384,385,386,387,388,389,390,\n 391,392,393,394,395,396,397,398,399,\n 400,401,402,403,404,405,406,407,408,\n 409,410,411,412,413,414,415,416,417,\n 418,419,420,421,422,423,424,425,426,\n 427,428,429,430,431,432,433,434,435,\n 436,437,438,439,440,441,442,443,444,\n 445,446,447,448,449,450,451,452,453,\n 454,455,456,457,458,459,460,461,462,\n 463,464,465,466,467,468,469,470,471,\n 472,473,474,475,476,477,478,479,480,\n 481,482,483,484,485,486,487,488,489,\n 490,491,492,493,494,495,496,497,498,\n 499,500,501,502,503,504,505,506,507,\n 508,509,510,511,512,513,514,515,516,\n 517,518,519,520,521,522,523,524,525,\n 526,527,528,529,530,531,532,533,534,\n 535,536,537,538,539,540,541,542,543,\n 544,545,546,547,548,549,550,551,552,\n 553,554,555,556,557,558,559,560,561,\n 562,563,564,565,566,567,568,569,570,\n 571,572,573,574,575,576,577,578,579,\n 580,581,582,583,584,585,586,587,588,\n 589,590,591,592,593,594,595,596,597,\n 598,599,600,601,602,603,604,605,606,\n 607,608,609,610,611,612,613,614,615,\n 616,617,618,619,620,621,622,623,624,\n 625,626,627,628,629,630,631,632,633,\n 634,635,636,637,638,639,640,641,642,\n 643,644,645,646,647,648,649,650,651,\n 652,653,654,655,656,657,658,659,660,\n 661,662,663,664,665,666,667,668,669,\n 670,671,672,673,674,675,676,677,678,\n 679,680,681,682,683,684,685,686,687,\n 688,689,690,691,692,693,694,695,696,\n 697,698,699,700,701,702,703,704,705,\n 706,707,708,709,710,711,712,713,714,\n 715,716,717,718,719,720,721,722,723,\n 724,725,726,727,728,729,730,731,732,\n 733,734,735,736,737,738,739,740,741,\n 742,743,744,745,746,747,748,749,750,\n 751,752,753,754,755,756,757,758,759,\n 760,761,762,763,764,765,766,767,768,\n 769,770,771,772,773,774,775,776,777,\n 778,779,780,781,782,783,784,785,786,\n 787,788,789,790,791,792,793,794,795,\n 796,797,798,799,800,801,802,803,804,\n 805,806,807,808,809,810,811,812,813,\n 814,815,816,817,818,819,820,821,822,\n 823,824,825,826,827,828,829,830,831,\n 832,833,834,835,836,837,838,839,840,\n 841,842,843,844,845,846,847,848,849,\n 850,851,852,853,939,940,941,942,943,\n 944,945,946,947,948,949,950,951,952,\n 953,954,955,956,957,958,959,960,961,\n 962,963,964,965,966,967,968,969,970,\n 971,972,973,974,975,976,977,978,979,\n 980,981,982,983,984,985,986,987,988,\n 989,990,991,992,993,994,995,996,997,\n 998,999,1000,1001,1002,1003,1004,\n 1005,1006,1007,1008,1009,1010,1011,\n 1012,1013,1014,1015,1016,1017,1018,\n 1019,1020,1021,1022,1023]},\n 360000]}}},\n [{janitor_agent,get_servant_call_reply,1,\n [{file,"src/janitor_agent.erl"},\n {line,532}]},\n {janitor_agent,\n get_mass_tap_docs_estimate,3,\n [{file,"src/janitor_agent.erl"},\n {line,554}]},\n {ns_rebalance_observer,\n \'-initiate_bucket_rebalance/2-fun-0-\',3,\n [{file,"src/ns_rebalance_observer.erl"},\n {line,195}]},\n {misc,\'-parallel_map/3-fun-0-\',4,\n [{file,"src/misc.erl"},{line,88}]}]}},\n [{{\'ns_1@172.23.107.34\',85},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',86},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',87},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',88},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',89},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',90},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',91},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',92},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',93},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',94},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',95},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',96},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',97},{7,4,tap}},\n {{\'ns_1@172.23.107.34\',98},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',99},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',100},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',101},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',102},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',103},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',104},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',105},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',106},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',107},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',108},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',109},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',110},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',111},{7,4,tap}},\n {{\'ns_1@172.23.107.34\',112},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',113},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',114},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',115},{7,4,tap}},\n {{\'ns_1@172.23.107.34\',116},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',117},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',118},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',119},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',120},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',121},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',122},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',123},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',124},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',125},{7,4,tap}},\n {{\'ns_1@172.23.107.34\',126},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',127},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',128},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',129},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',130},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',131},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',132},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',133},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',134},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',135},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',136},{7,4,tap}},\n {{\'ns_1@172.23.107.34\',137},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',138},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',139},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',140},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',141},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',142},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',143},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',144},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',145},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',146},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',147},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',148},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',149},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',150},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',151},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',152},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',153},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',154},{7,4,tap}},\n {{\'ns_1@172.23.107.34\',155},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',156},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',157},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',158},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',159},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',160},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',161},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',162},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',163},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',164},{7,4,tap}},\n {{\'ns_1@172.23.107.34\',165},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',166},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',167},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',168},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',169},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',170},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',171},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',172},{2,1,tap}},\n {{\'ns_1@172.23.107.34\',173},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',174},{2,1,tap}},\n {{\'ns_1@172.23.107.34\',175},{2,1,tap}},\n {{\'ns_1@172.23.107.34\',176},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',177},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',178},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',179},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',180},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',181},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',182},{4,1,tap}},\n {{\'ns_1@172.23.107.34\',183},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',184},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',185},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',186},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',187},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',188},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',189},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',190},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',191},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',192},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',193},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',194},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',195},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',196},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',197},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',198},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',199},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',200},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',201},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',202},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',203},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',204},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',205},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',206},{7,4,tap}},\n {{\'ns_1@172.23.107.34\',207},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',208},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',209},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',210},{7,4,tap}},\n {{\'ns_1@172.23.107.34\',211},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',212},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',213},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',214},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',215},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',216},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',217},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',218},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',219},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',220},{7,4,tap}},\n {{\'ns_1@172.23.107.34\',221},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',222},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',223},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',224},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',225},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',226},{4,1,tap}},\n {{\'ns_1@172.23.107.34\',227},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',228},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',229},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',230},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',231},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',232},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',233},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',234},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',235},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',236},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',237},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',238},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',239},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',240},{7,4,tap}},\n {{\'ns_1@172.23.107.34\',241},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',242},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',243},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',244},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',245},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',246},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',247},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',248},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',249},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',250},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',251},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',252},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',253},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',254},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',255},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',512},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',513},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',514},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',515},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',516},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',517},{7,4,tap}},\n {{\'ns_1@172.23.107.34\',518},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',519},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',520},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',521},{7,4,tap}},\n {{\'ns_1@172.23.107.34\',522},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',523},{7,4,tap}},\n {{\'ns_1@172.23.107.34\',524},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',525},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',526},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',527},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',528},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',529},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',530},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',531},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',532},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',533},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',534},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',535},{7,4,tap}},\n {{\'ns_1@172.23.107.34\',536},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',537},{7,4,tap}},\n {{\'ns_1@172.23.107.34\',538},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',539},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',540},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',541},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',542},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',543},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',544},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',545},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',546},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',547},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',548},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',549},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',550},{7,4,tap}},\n {{\'ns_1@172.23.107.34\',551},{7,4,tap}},\n {{\'ns_1@172.23.107.34\',552},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',553},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',554},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',555},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',556},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',557},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',558},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',559},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',560},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',561},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',562},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',563},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',564},{7,4,tap}},\n {{\'ns_1@172.23.107.34\',565},{7,4,tap}},\n {{\'ns_1@172.23.107.34\',566},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',567},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',568},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',569},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',570},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',571},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',572},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',573},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',574},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',575},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',576},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',577},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',578},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',579},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',580},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',581},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',582},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',583},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',584},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',585},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',586},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',587},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',588},{7,4,tap}},\n {{\'ns_1@172.23.107.34\',589},{7,4,tap}},\n {{\'ns_1@172.23.107.34\',590},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',591},{3,2,tap}},\n {{\'ns_1@172.23.107.34\',592},{0,0,tap}},\n {{\'ns_1@172.23.107.34\',593},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',594},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',595},{5,3,tap}},\n {{\'ns_1@172.23.107.34\',596},{0,0,tap}}]],\n []},\n {lists,append,1,\n [{file,"lists.erl"},{line,125}]},\n {lists,append,1,\n [{file,"lists.erl"},{line,125}]},\n {ns_rebalance_observer,\n initiate_bucket_rebalance,2,\n [{file,"src/ns_rebalance_observer.erl"},\n {line,209}]},\n {ns_rebalance_observer,handle_set_ff_map,2,\n [{file,"src/ns_rebalance_observer.erl"},\n {line,257}]},\n {ns_rebalance_observer,handle_cast,2,\n [{file,"src/ns_rebalance_observer.erl"},\n {line,105}]},\n {gen_server,handle_msg,5,\n [{file,"gen_server.erl"},{line,604}]},\n {proc_lib,init_p_do_apply,3,\n [{file,"proc_lib.erl"},{line,239}]}]}\n', u'shortText': u'message', u'serverTime': u'2016-05-19T15:46:53.776Z', u'module': u'ns_orchestrator', u'tstamp': 1463698013776, u'type': u'info'}
|
2016-05-19 15:47:03,570 - root - ERROR - {u'node': u'ns_1@172.23.107.143', u'code': 0, u'text': u"Control connection to memcached on 'ns_1@172.23.107.143' disconnected: {error,\n closed}", u'shortText': u'message', u'serverTime': u'2016-05-19T15:46:53.775Z', u'module': u'ns_memcached', u'tstamp': 1463698013775, u'type': u'info'}
|
2016-05-19 15:47:03,570 - root - ERROR - {u'node': u'ns_1@172.23.105.108', u'code': 0, u'text': u'Bucket "default" loaded on node \'ns_1@172.23.105.108\' in 0 seconds.', u'shortText': u'message', u'serverTime': u'2016-05-19T15:46:53.681Z', u'module': u'ns_memcached', u'tstamp': 1463698013681, u'type': u'info'}
|
2016-05-19 15:47:03,571 - root - ERROR - {u'node': u'ns_1@172.23.105.108', u'code': 0, u'text': u"Control connection to memcached on 'ns_1@172.23.105.108' disconnected: {error,\n closed}", u'shortText': u'message', u'serverTime': u'2016-05-19T15:46:53.675Z', u'module': u'ns_memcached', u'tstamp': 1463698013675, u'type': u'info'}
|
2016-05-19 15:47:03,571 - root - ERROR - {u'node': u'ns_1@172.23.105.108', u'code': 0, u'text': u'Bucket "default" rebalance does not seem to be swap rebalance', u'shortText': u'message', u'serverTime': u'2016-05-19T15:46:53.662Z', u'module': u'ns_vbucket_mover', u'tstamp': 1463698013662, u'type': u'info'}
|
2016-05-19 15:47:03,571 - root - ERROR - {u'node': u'ns_1@172.23.105.108', u'code': 0, u'text': u'Started rebalancing bucket default', u'shortText': u'message', u'serverTime': u'2016-05-19T15:46:53.538Z', u'module': u'ns_rebalancer', u'tstamp': 1463698013538, u'type': u'info'}
|
2016-05-19 15:47:03,571 - root - ERROR - {u'node': u'ns_1@172.23.105.108', u'code': 4, u'text': u"Starting rebalance, KeepNodes = ['ns_1@172.23.107.143','ns_1@172.23.105.108'], EjectNodes = ['ns_1@172.23.106.57',\n 'ns_1@172.23.107.34'], Failed over and being ejected nodes = []; no delta recovery nodes\n", u'shortText': u'message', u'serverTime': u'2016-05-19T15:46:53.502Z', u'module': u'ns_orchestrator', u'tstamp': 1463698013502, u'type': u'info'}
|
2016-05-19 15:47:03,571 - root - ERROR - {u'node': u'ns_1@172.23.105.108', u'code': 0, u'text': u'Reset auto-failover count', u'shortText': u'message', u'serverTime': u'2016-05-19T15:46:36.279Z', u'module': u'auto_failover', u'tstamp': 1463697996279, u'type': u'info'}
|
2016-05-19 15:47:03,572 - root - ERROR - {u'node': u'ns_1@172.23.105.108', u'code': 1, u'text': u'Rebalance completed successfully.\n', u'shortText': u'message', u'serverTime': u'2016-05-19T15:46:36.271Z', u'module': u'ns_orchestrator', u'tstamp': 1463697996271, u'type': u'info'}
|
Retried rebalance again 3 times. Still failed.
