Details
-
Bug
-
Resolution: Incomplete
-
Critical
-
6.5.0
-
Triaged
-
Unknown
Description
Build: 6.5.0-3511
Test: eventing component
Day: 2nd
Cycle: 6th
Test Step : failover -> delta recovery of kv node
[2019-06-18T08:26:36-07:00, sequoiatools/couchbase-cli:6.5:68a632] failover -c 172.23.104.16:8091 --server-failover 172.23.104.18:8091 -u Administrator -p password --force
|
[2019-06-18T08:26:54-07:00, sequoiatools/couchbase-cli:6.5:e17fbf] recovery -c 172.23.104.16:8091 --server-recovery 172.23.104.18:8091 --recovery-type delta -u Administrator -p password
|
[2019-06-18T08:27:02-07:00, sequoiatools/couchbase-cli:6.5:6f0e49] rebalance -c 172.23.104.16:8091 -u Administrator -p password
|
→
|
|
|
Error occurred on container - sequoiatools/couchbase-cli:6.5:[rebalance -c 172.23.104.16:8091 -u Administrator -p password]
|
|
|
docker logs 6f0e49
|
docker start 6f0e49
|
|
|
*Unable to display progress bar on this os
|
JERROR: Rebalance failed. See logs for detailed reason. You can try again.
|
[2019-06-18T08:30:10-07:00, sequoiatools/cmd:0492b0] 60
|
Error:
[ns_server:error,2019-06-18T08:28:45.563-07:00,ns_1@172.23.104.16:ns_doctor<0.10641.0>:ns_doctor:update_status:316]The following buckets became not ready on node 'ns_1@172.23.104.16': ["bucket_op",
|
"bucket_op_dst",
|
"curl_op_dst",
|
"metadata",
|
"source_op",
|
"test_diagnostics",
|
"timer_op",
|
"timer_op_dst"], those of them are active ["bucket_op",
|
"bucket_op_dst",
|
"curl_op_dst",
|
"metadata",
|
"source_op",
|
"test_diagnostics",
|
"timer_op",
|
"timer_op_dst"]
|
[ns_server:error,2019-06-18T08:30:03.406-07:00,ns_1@172.23.104.16:<0.32289.978>:ns_rebalancer:get_active_failover_logs:2000]Failed to get failover logs from some nodes for delta recovery.
|
Bucket: "timer_op"
|
Requests: [{'ns_1@172.23.104.16',[562,561,560,559,558,557,556,555,554,553,552,
|
551,550,549,548,547,546,545,544,543,542,541,
|
540,539,538,537,536,535,534,533,532,531,530,
|
529,528,527,526,525,524,523,522,521,520,519,
|
518,517,516,515,514,513,512,376,375,374,373,
|
372,371,370,369,368,367,366,365,364,363,362,
|
361,360,359,358,357,356,355,354,353,352,351,
|
350,349,348,347,346,345,344,343,342,290,289,
|
288,287,286,285,284,283,282,281,280,279,278,
|
277,276,275,274,273,272,271,270,269,268,267,
|
266,265,264,263,262,261,260,259,258,257,256,
|
136,135,134,133,132,131,130,129,128,127,126,
|
125,124,123,122,121,120,119,118,117,116,115,
|
114,113,112,111,110,109,108,107,106,105,104,
|
103,102,101,100,99,98,97,96,95,94,93,92,91,
|
90,89,88,87,86]},
|
{'ns_1@172.23.104.19',[989,988,987,986,985,984,983,982,981,980,979,
|
978,977,976,975,974,973,972,971,970,969,968,
|
967,966,965,964,963,962,961,960,959,958,957,
|
956,955,954,953,952,951,950,949,948,947,946,
|
945,944,943,942,941,940,939,938,867,866,865,
|
864,863,862,861,860,859,858,857,856,855,854,
|
853,733,732,731,730,729,728,727,726,725,724,
|
723,722,721,720,719,718,717,716,715,714,713,
|
712,711,710,709,708,707,706,705,704,703,702,
|
701,700,699,698,697,696,695,694,693,692,691,
|
690,689,688,687,686,685,684,683,682,647,612,
|
611,610,609,608,607,606,605,604,603,602,601,
|
600,599,598,597,477,476,475,474,473,472,471,
|
470,469,468,467,466,465,464,463,462,461,392,
|
391,390,389,388,387,386,385,384,383,382,381,
|
380,379,378,377,51]},
|
{'ns_1@172.23.104.25',[1023,1022,1021,1020,1019,1018,1017,1016,
|
1015,1014,1013,1012,1011,1010,1009,1008,
|
1007,1006,1005,1004,1003,1002,1001,1000,999,
|
998,997,996,995,994,993,992,991,990,903,680,
|
679,678,677,676,675,674,673,672,671,670,669,
|
668,667,666,665,664,663,662,661,660,659,658,
|
657,656,655,654,653,652,651,650,649,648,646,
|
645,644,643,642,641,640,639,638,637,636,635,
|
634,633,632,631,630,629,628,627,626,625,624,
|
623,622,621,620,619,618,617,616,615,614,613,
|
426,425,424,423,422,421,420,419,418,417,416,
|
415,414,413,412,411,410,409,408,407,406,405,
|
404,403,402,401,400,399,398,397,396,395,394,
|
393,170,169,168,167,166,165,164,163,162,161,
|
160,159,158,157,156,155,154,153,152,151,150,
|
149,148,147,146,145,144,143,142,141,140,139,
|
138,137]}]
|
Errors: {error,{failed_nodes,[{'ns_1@172.23.104.16',{error,timeout}}]}}[user:error,2019-06-18T08:30:03.408-07:00,ns_1@172.23.104.16:<0.10690.0>:ns_orchestrator:log_rebalance_completion:1426]Rebalance exited with reason {get_failover_logs_failed,"timer_op",
|
[{'ns_1@172.23.104.16',
|
[562,561,560,559,558,557,556,555,554,553,552,
|
551,550,549,548,547,546,545,544,543,542,541,
|
540,539,538,537,536,535,534,533,532,531,530,
|
529,528,527,526,525,524,523,522,521,520,519,
|
518,517,516,515,514,513,512,376,375,374,373,
|
372,371,370,369,368,367,366,365,364,363,362,
|
361,360,359,358,357,356,355,354,353,352,351,
|
350,349,348,347,346,345,344,343,342,290,289,
|
288,287,286,285,284,283,282,281,280,279,278,
|
277,276,275,274,273,272,271,270,269,268,267,
|
266,265,264,263,262,261,260,259,258,257,256,
|
136,135,134,133,132,131,130,129,128,127,126,
|
125,124,123,122,121,120,119,118,117,116,115,
|
114,113,112,111,110,109,108,107,106,105,104,
|
103,102,101,100,99,98,97,96,95,94,93,92,91,
|
90,89,88,87,86]},
|
{'ns_1@172.23.104.19',
|
[989,988,987,986,985,984,983,982,981,980,979,
|
978,977,976,975,974,973,972,971,970,969,968,
|
967,966,965,964,963,962,961,960,959,958,957,
|
956,955,954,953,952,951,950,949,948,947,946,
|
945,944,943,942,941,940,939,938,867,866,865,
|
864,863,862,861,860,859,858,857,856,855,854,
|
853,733,732,731,730,729,728,727,726,725,724,
|
723,722,721,720,719,718,717,716,715,714,713,
|
712,711,710,709,708,707,706,705,704,703,702,
|
701,700,699,698,697,696,695,694,693,692,691,
|
690,689,688,687,686,685,684,683,682,647,612,
|
611,610,609,608,607,606,605,604,603,602,601,
|
600,599,598,597,477,476,475,474,473,472,471,
|
470,469,468,467,466,465,464,463,462,461,392,
|
391,390,389,388,387,386,385,384,383,382,381,
|
380,379,378,377,51]},
|
{'ns_1@172.23.104.25',
|
[1023,1022,1021,1020,1019,1018,1017,1016,1015,
|
1014,1013,1012,1011,1010,1009,1008,1007,1006,
|
1005,1004,1003,1002,1001,1000,999,998,997,
|
996,995,994,993,992,991,990,903,680,679,678,
|
677,676,675,674,673,672,671,670,669,668,667,
|
666,665,664,663,662,661,660,659,658,657,656,
|
655,654,653,652,651,650,649,648,646,645,644,
|
643,642,641,640,639,638,637,636,635,634,633,
|
632,631,630,629,628,627,626,625,624,623,622,
|
621,620,619,618,617,616,615,614,613,426,425,
|
424,423,422,421,420,419,418,417,416,415,414,
|
413,412,411,410,409,408,407,406,405,404,403,
|
402,401,400,399,398,397,396,395,394,393,170,
|
169,168,167,166,165,164,163,162,161,160,159,
|
158,157,156,155,154,153,152,151,150,149,148,
|
147,146,145,144,143,142,141,140,139,138,
|
137]}],
|
{error,
|
{failed_nodes,
|
[{'ns_1@172.23.104.16',{error,timeout}}]}}}.
|
Rebalance Operation Id = 1edcbcd06933e47addb1da11b9b8a4fc
|
[ns_server:error,2019-06-18T08:30:39.112-07:00,ns_1@172.23.104.16:<0.14108.0>:ns_memcached:worker_init:183]Unexpected exception in ns_memcached for bucket "bucket_op": {error,
|
{badmatch,
|
{error,
|
timeout}},
|
[{mc_client_binary,
|
stats_recv,4,
|
[{file,
|
"src/mc_client_binary.erl"},
|
{line,165}]},
|
{mc_client_binary,
|
stats,4,
|
[{file,
|
"src/mc_client_binary.erl"},
|
{line,404}]},
|
{ns_memcached,
|
do_handle_call,
|
3,
|
[{file,
|
"src/ns_memcached.erl"},
|
{line,444}]},
|
{ns_memcached,
|
worker_loop,3,
|
[{file,
|
"src/ns_memcached.erl"},
|
{line,220}]},
|
{ns_memcached,
|
worker_init,2,
|
[{file,
|
"src/ns_memcached.erl"},
|
{line,179}]},
|
{proc_lib,
|
init_p_do_apply,
|
3,
|
[{file,
|
"proc_lib.erl"},
|
{line,
|
247}]}]}
|
[ns_server:error,2019-06-18T08:30:39.112-07:00,ns_1@172.23.104.16:<0.17372.0>:ns_memcached:worker_init:183]Unexpected exception in ns_memcached for bucket "curl_op_dst": {error,
|
{badmatch,
|
{error,
|
timeout}},
|
[{mc_client_binary,
|
stats_recv,
|
4,
|