Details
-
Bug
-
Resolution: Fixed
-
Critical
-
3.0
-
Security Level: Public
-
None
-
3.0.0-893
-
Untriaged
-
Centos 64-bit
-
Unknown
Description
rebalance 3+1 node on destination cluster
seen the same stack in MB-10936(couchbase-bucket team) but in my case I don't see (upr_add_stream) vbucket = 0 opaque = 0x20 status = 0x23 (rollback) in logs so don't know whose this issue
Rebalance exited with reason {unexpected_exit,
{'EXIT',<0.25286.9>,
{upr_wait_for_data_move_failed,"AbRegNums",
510,'ns_1@172.23.105.160',
['ns_1@172.23.105.159','ns_1@172.23.105.207'],
{error,no_stats_for_this_vbucket}}}}
[rebalance:debug,2014-06-28T13:18:40.592,ns_1@172.23.105.159:<0.25286.9>:ns_single_vbucket_mover:wait_upr_data_move:313]Will wait for backfill on all opened streams for bucket = "AbRegNums" partition 510 src node = 'ns_1@172.23.105.160' dest nodes = ['ns_1@172.23.105.159',
'ns_1@172.23.105.207']
[ns_server:debug,2014-06-28T13:18:40.593,ns_1@172.23.105.159:<0.25281.9>:upr_commands:open_connection:53]Open consumer connection "replication:ns_1@172.23.105.160->ns_1@172.23.105.159:AbRegNums" on socket #Port<0.49741>
[ns_server:debug,2014-06-28T13:18:40.594,ns_1@172.23.105.159:<0.25282.9>:upr_commands:open_connection:53]Open producer connection "replication:ns_1@172.23.105.160->ns_1@172.23.105.159:AbRegNums" on socket #Port<0.49742>
[rebalance:info,2014-06-28T13:18:40.596,ns_1@172.23.105.159:<0.25215.9>:janitor_agent:get_vbucket_high_seqno:449]AbRegNums: Doing get_vbucket_high_seqno call for vbucket 1022 on ns_1@172.23.105.206
[ns_server:debug,2014-06-28T13:18:40.595,ns_1@172.23.105.159:upr_replicator-AbRegNums-ns_1@172.23.105.160<0.7020.3>:upr_replicator:terminate:100]Terminating with reason {{badmatch,
[{<0.25277.9>,
{done,exit,
{normal,
{gen_server,call,
[<0.7021.3>,
infinity]}},
[{gen_server,call,3,
[{file,"gen_server.erl"},{line,188}]},
{upr_replicator,'
[{file,"src/upr_replicator.erl"},{line,119}]},
{upr_replicator,'
[{file,"src/upr_replicator.erl"},
{line,189}]}]}}]},
[{misc,sync_shutdown_many_i_am_trapping_exits,1,
[{file,"src/misc.erl"},{line,1475}]},
{upr_replicator,spawn_and_wait,1,
[{file,"src/upr_replicator.erl"},{line,211}]},
{upr_replicator,handle_call,3,
[{file,"src/upr_replicator.erl"},{line,118}]},
{gen_server,handle_msg,5,
[{file,"gen_server.erl"},{line,585}]},
{proc_lib,init_p_do_apply,3,
[{file,"proc_lib.erl"},{line,239}]}]}. Nuked connection "replication:ns_1@172.23.105.160->ns_1@172.23.105.159:AbRegNums" with result [ok,
ok].
[error_logger:error,2014-06-28T13:18:40.596,ns_1@172.23.105.159:error_logger<0.6.0>:ale_error_logger_handler:do_log:203]** Generic server 'upr_replicator-AbRegNums-ns_1@172.23.105.160' terminating
** Last message in was {takeover,511}
** When Server state == {state,<0.7022.3>,<0.7021.3>, "replication:ns_1@172.23.105.160->ns_1@172.23.105.159:AbRegNums", 'ns_1@172.23.105.160',"AbRegNums"}
** Reason for termination ==
** {{badmatch,
[{<0.25277.9>,
{done,exit,
{normal,
{gen_server,call,
[<0.7021.3>,{maybe_close_stream,511}
,infinity]}},
[{gen_server,call,3,[
{upr_replicator,'
[{file,"src/upr_replicator.erl"},{line,119}]},
{upr_replicator,'
[{file,"src/upr_replicator.erl"},{line,189}]}]}}]},
[{misc,sync_shutdown_many_i_am_trapping_exits,1,
[{file,"src/misc.erl"},{line,1475}]},
{upr_replicator,spawn_and_wait,1,
[{file,"src/upr_replicator.erl"},{line,211}]},
{upr_replicator,handle_call,3,
[{file,"src/upr_replicator.erl"},{line,118}]},
{gen_server,handle_msg,5,[{file,"gen_server.erl"}
,
{line,585}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,239}]}]}
[ns_server:debug,2014-06-28T13:18:40.596,ns_1@172.23.105.159:replication_manager-AbRegNums<0.6950.3>:replication_manager:terminate:103]Replication manager died {{{{badmatch,
[{<0.25277.9>,
{done,exit,
{normal,
{gen_server,call,
[<0.7021.3>,
{maybe_close_stream,511},
infinity]}},
[{gen_server,call,3,
[{file,"gen_server.erl"},{line,188}]},
{upr_replicator,'
[{file,"src/upr_replicator.erl"},
{line,119}]},
{upr_replicator,'
1,
[{file,"src/upr_replicator.erl"},
{line,189}]}]}}]},
[{misc,sync_shutdown_many_i_am_trapping_exits,1,
[{file,"src/misc.erl"},{line,1475}]},
{upr_replicator,spawn_and_wait,1,
[{file,"src/upr_replicator.erl"},{line,211}]},
{upr_replicator,handle_call,3,
[{file,"src/upr_replicator.erl"},{line,118}]},
{gen_server,handle_msg,5,
[{file,"gen_server.erl"},{line,585}
]},
{proc_lib,init_p_do_apply,3,
[
{gen_server,call,
['upr_replicator-AbRegNums-ns_1@172.23.105.160',
{takeover,511},
infinity]}},
{state,"AbRegNums",upr,
[{'ns_1@172.23.105.160', [342,343,344,345,346,347,348,349,350,351,352,353, 354,355,356,357,358,359,360,361,362,363,364,365, 366,367,368,369,370,371,372,373,374,375,376,377, 378,379,380,381,382,383,384,385,386,387,388,389, 390,391,392,393,394,395,396,397,398,399,400,401, 402,403,404,405,406,407,408,409,410,411,412,413, 414,415,416,417,418,419,420,421,422,423,424,425, 426,427,428,429,430,431,432,433,434,435,436,437, 438,439,440,441,442,443,444,445,446,447,448,449, 450,451,452,453,454,455,456,457,458,459,460,461, 462,463,464,465,466,467,468,469,470,471,472,473, 474,475,476,477,478,479,480,481,482,483,484,485, 486,487,488,489,490,491,492,493,494,495,496,497, 498,499,500,501,502,503,504,505,506,507,508,509, 510,511,512]},
{'ns_1@172.23.105.206', [683,684,685,686,687,688,689,690,691,692,693,694, 695,696,697,698,699,700,701,702,703,704,705,706, 707,708,709,710,711,712,713,714,715,716,717,718, 719,720,721,722,723,724,725,726,727,728,729,730, 731,732,733,734,735,736,737,738,739,740,741,742, 743,744,745,746,747,748,749,750,751,752,753,754, 755,756,757,758,759,760,761,762,763,764,765,766, 767,768,769,770,771,772,773,774,775,776,777,778, 779,780,781,782,783,784,785,786,787,788,789,790, 791,792,793,794,795,796,797,798,799,800,801,802, 803,804,805,806,807,808,809,810,811,812,813,814, 815,816,817,818,819,820,821,822,823,824,825,826, 827,828,829,830,831,832,833,834,835,836,837,838, 839,840,841,842,843,844,845,846,847,848,849,850, 851,852,853]}],
undefined}}
[rebalance:info,2014-06-28T13:18:40.597,ns_1@172.23.105.159:<0.25173.9>:janitor_agent:get_vbucket_high_seqno:449]AbRegNums: Doing get_vbucket_high_seqno call for vbucket 512 on ns_1@172.23.105.160
[ns_server:error,2014-06-28T13:18:40.597,ns_1@172.23.105.159:<0.25239.9>:ns_single_vbucket_mover:spawn_and_wait:112]Got unexpected exit signal {'EXIT',<0.25286.9>,
{upr_wait_for_data_move_failed,"AbRegNums",
510,'ns_1@172.23.105.160',
['ns_1@172.23.105.159',
'ns_1@172.23.105.207'],
{error,no_stats_for_this_vbucket}}}
[ns_server:error,2014-06-28T13:18:40.598,ns_1@172.23.105.159:<0.25239.9>:misc:sync_shutdown_many_i_am_trapping_exits:1473]Shutdown of the following failed: [{<0.25286.9>,
{upr_wait_for_data_move_failed,
"AbRegNums",510,'ns_1@172.23.105.160',
['ns_1@172.23.105.159',
'ns_1@172.23.105.207'],
{error,no_stats_for_this_vbucket}}}]
[ns_server:debug,2014-06-28T13:18:40.598,ns_1@172.23.105.159:<0.6953.3>:ns_process_registry:handle_info:109]'rebalance_subprocesses_registry-AbRegNums' detected abnormal exit: {'EXIT',
<0.25276.9>,
{{{{badmatch,
[{<0.25277.9>,
{done,
exit,
{normal,
{gen_server,
call,
[<0.7021.3>,
{maybe_close_stream, 511},
infinity]}},
[{gen_server,
call,
3,
[{file, "gen_server.erl"},
{line, 188}]},
{upr_replicator,
'
2,
[{file, "src/upr_replicator.erl"},
{line, 119}]},
{upr_replicator,
'
1,
[{file, "src/upr_replicator.erl"},
{line, 189}]}]}}]},
[{misc,
sync_shutdown_many_i_am_trapping_exits,
1,
[{file, "src/misc.erl"},
{line, 1475}]},
{upr_replicator,
spawn_and_wait,
1,
[{file, "src/upr_replicator.erl"},
{line, 211}]},
{upr_replicator,
handle_call,
3,
[{file, "src/upr_replicator.erl"},
{line, 118}]},
{gen_server,
handle_msg,
5,
[{file, "gen_server.erl"},
{line, 585}]},
{proc_lib,
init_p_do_apply,
3,
[{file, "proc_lib.erl"},
{line, 239}]}]},
{gen_server,
call,
['upr_replicator-AbRegNums-ns_1@172.23.105.160',
{takeover, 511},
infinity]}},
{gen_server,
call,
['replication_manager-AbRegNums',
{upr_takeover, 'ns_1@172.23.105.160', 511},
infinity]}}}
[ns_server:error,2014-06-28T13:18:40.598,ns_1@172.23.105.159:<0.25203.9>:ns_single_vbucket_mover:spawn_and_wait:112]Got unexpected exit signal {'EXIT',<0.25275.9>,
{{{{{badmatch,
[{<0.25277.9>,
{done,exit,
{normal,
{gen_server,call,
[<0.7021.3>,
{maybe_close_stream,511},
infinity]}},
[{gen_server,call,3,
[{file,"gen_server.erl"},{line,188}]},
{upr_replicator,'
2,
[{file,"src/upr_replicator.erl"},
{line,119}]},
{upr_replicator,
'
[{file,"src/upr_replicator.erl"},
{line,189}]}]}}]},
[{misc,
sync_shutdown_many_i_am_trapping_exits,1,
[{file,"src/misc.erl"},{line,1475}]},
{upr_replicator,spawn_and_wait,1,
[{file,"src/upr_replicator.erl"},
{line,211}]},
{upr_replicator,handle_call,3,
[{file,"src/upr_replicator.erl"},
{line,118}]},
{gen_server,handle_msg,5,
[{file,"gen_server.erl"},{line,585}]},
{proc_lib,init_p_do_apply,3,
[{file,"proc_lib.erl"}
,
{line,239}]}]},{gen_server,call,
['upr_replicator-AbRegNums-ns_1@172.23.105.160',
{takeover,511},
infinity]}},
{gen_server,call,
['replication_manager-AbRegNums',
{upr_takeover,'ns_1@172.23.105.160',511},
infinity]}},
{gen_server,call,
[{'janitor_agent-AbRegNums', 'ns_1@172.23.105.159'},
{if_rebalance,<0.25155.9>,
{upr_takeover,'ns_1@172.23.105.160',511}},
infinity]}}}
[ns_server:error,2014-06-28T13:18:40.599,ns_1@172.23.105.159:<0.25239.9>:misc:try_with_maybe_ignorant_after:1509]Eating exception from ignorant after-block:
{error,
{badmatch,
[{<0.25286.9>,
{upr_wait_for_data_move_failed,"AbRegNums",510,
'ns_1@172.23.105.160',
['ns_1@172.23.105.159','ns_1@172.23.105.207'],
{error,no_stats_for_this_vbucket}}}]},
[{misc,sync_shutdown_many_i_am_trapping_exits,1,
[{file,"src/misc.erl"},{line,1475}]},
{misc,try_with_maybe_ignorant_after,2,
[{file,"src/misc.erl"},{line,1507}]},
{ns_single_vbucket_mover,mover,6,
[{file,"src/ns_single_vbucket_mover.erl"},{line,77}]},
{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,239}
]}]}
[rebalance:error,2014-06-28T13:18:40.600,ns_1@172.23.105.159:<0.25155.9>:ns_vbucket_mover:handle_info:198]<0.25239.9> exited with {unexpected_exit,
{'EXIT',<0.25286.9>,
{upr_wait_for_data_move_failed,"AbRegNums",
510,'ns_1@172.23.105.160',
['ns_1@172.23.105.159',
'ns_1@172.23.105.207'],
{error,no_stats_for_this_vbucket}}}}
[ns_server:error,2014-06-28T13:18:40.600,ns_1@172.23.105.159:<0.25203.9>:misc:sync_shutdown_many_i_am_trapping_exits:1473]Shutdown of the following failed: [{<0.25275.9>,
{{{{{badmatch,
[{<0.25277.9>,
{done,exit,
{normal,
{gen_server,call,
[<0.7021.3>,
infinity]}},
[{gen_server,call,3,
[{file,"gen_server.erl"},
{line,188}]},
{upr_replicator,
'
[{file,"src/upr_replicator.erl"},
{line,119}]},
{upr_replicator,
'
[{file,"src/upr_replicator.erl"},
{line,189}]}]}}]},
[{misc,
sync_shutdown_many_i_am_trapping_exits,
1,
[{file,"src/misc.erl"},{line,1475}]},
{upr_replicator,spawn_and_wait,1,
[{file,"src/upr_replicator.erl"},
{line,211}]},
{upr_replicator,handle_call,3,
[{file,"src/upr_replicator.erl"},
{line,118}]},
{gen_server,handle_msg,5,
[{file,"gen_server.erl"},
{line,585}]},
{proc_lib,init_p_do_apply,3,
[{file,"proc_lib.erl"},
{line,239}]}]},
{gen_server,call,
['upr_replicator-AbRegNums-ns_1@172.23.105.160',
{takeover,511},
infinity]}},
{gen_server,call,
['replication_manager-AbRegNums',
{upr_takeover,'ns_1@172.23.105.160', 511},
infinity]}},
{gen_server,call,
[{'janitor_agent-AbRegNums', 'ns_1@172.23.105.159'},
{if_rebalance,<0.25155.9>,
{upr_takeover,'ns_1@172.23.105.160',
511}},
infinity]}}}]
[ns_server:error,2014-06-28T13:18:40.600,ns_1@172.23.105.159:<0.25176.9>:ns_single_vbucket_mover:spawn_and_wait:112]Got unexpected exit signal {'EXIT',<0.25155.9>,
{unexpected_exit,
{'EXIT',<0.25286.9>,
{upr_wait_for_data_move_failed,"AbRegNums",510,
'ns_1@172.23.105.160',
['ns_1@172.23.105.159','ns_1@172.23.105.207'],
{error,no_stats_for_this_vbucket}}}}}
[ns_server:error,2014-06-28T13:18:40.601,ns_1@172.23.105.159:<0.25256.9>:ns_single_vbucket_mover:spawn_and_wait:112]Got unexpected exit signal {'EXIT',<0.25155.9>,
{unexpected_exit,
{'EXIT',<0.25286.9>,
{upr_wait_for_data_move_failed,"AbRegNums",510,
'ns_1@172.23.105.160',
['ns_1@172.23.105.159','ns_1@172.23.105.207'],
{error,no_stats_for_this_vbucket}}}}}
[ns_server:error,2014-06-28T13:18:40.601,ns_1@172.23.105.159:<0.25173.9>:ns_single_vbucket_mover:spawn_and_wait:112]Got unexpected exit signal {'EXIT',<0.25155.9>,
{unexpected_exit,
{'EXIT',<0.25286.9>,
{upr_wait_for_data_move_failed,"AbRegNums",510,
'ns_1@172.23.105.160',
['ns_1@172.23.105.159','ns_1@172.23.105.207'],
{error,no_stats_for_this_vbucket}}}}}
[ns_server:info,2014-06-28T13:18:40.601,ns_1@172.23.105.159:janitor_agent-AbRegNums<0.25295.9>:janitor_agent:read_flush_counter:943]Loading flushseq failed: {error,enoent}. Assuming it's equal to global config.
[ns_server:info,2014-06-28T13:18:40.602,ns_1@172.23.105.159:janitor_agent-AbRegNums<0.25295.9>:janitor_agent:read_flush_counter_from_config:950]Initialized flushseq 0 from bucket config
[ns_server:error,2014-06-28T13:18:40.602,ns_1@172.23.105.159:<0.25215.9>:ns_single_vbucket_mover:spawn_and_wait:112]Got unexpected exit signal {'EXIT',<0.25155.9>,
{unexpected_exit,
{'EXIT',<0.25286.9>,
{upr_wait_for_data_move_failed,"AbRegNums",510,
'ns_1@172.23.105.160',
['ns_1@172.23.105.159','ns_1@172.23.105.207'],
{error,no_stats_for_this_vbucket}}}}}
[ns_server:error,2014-06-28T13:18:40.601,ns_1@172.23.105.159:<0.25203.9>:misc:try_with_maybe_ignorant_after:1509]Eating exception from ignorant after-block:
{error,
{badmatch,
[{<0.25275.9>,
{{{{{badmatch,
[{<0.25277.9>,
{done,exit,
{normal,
{gen_server,call,
[<0.7021.3>,
{maybe_close_stream,511}
,
infinity]}},
[{gen_server,call,3,
[
{upr_replicator,'
[{file,"src/upr_replicator.erl"},{line,119}]},
{upr_replicator,'
[{file,"src/upr_replicator.erl"},
{line,189}]}]}}]},
[{misc,sync_shutdown_many_i_am_trapping_exits,1,
[{file,"src/misc.erl"},{line,1475}]},
{upr_replicator,spawn_and_wait,1,
[{file,"src/upr_replicator.erl"},{line,211}]},
{upr_replicator,handle_call,3,
[{file,"src/upr_replicator.erl"},{line,118}]},
{gen_server,handle_msg,5,[{file,"gen_server.erl"}
,
{line,585}]},{proc_lib,init_p_do_apply,3,
[{file,"proc_lib.erl"},{line,239}]}]},
{gen_server,call,
['upr_replicator-AbRegNums-ns_1@172.23.105.160',
{takeover,511},
infinity]}},
{gen_server,call,
['replication_manager-AbRegNums',
{upr_takeover,'ns_1@172.23.105.160',511},
infinity]}},
{gen_server,call,
[{'janitor_agent-AbRegNums','ns_1@172.23.105.159'},
{if_rebalance,<0.25155.9>,
{upr_takeover,'ns_1@172.23.105.160',511}},
infinity]}}}]},
[{misc,sync_shutdown_many_i_am_trapping_exits,1,
[{file,"src/misc.erl"},{line,1475}]},
{misc,try_with_maybe_ignorant_after,2,
[{file,"src/misc.erl"},{line,1507}]},
{ns_single_vbucket_mover,mover,6,
[{file,"src/ns_single_vbucket_mover.erl"},{line,77}]},
{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,239}]}]}
[error_logger:error,2014-06-28T13:18:40.602,ns_1@172.23.105.159:error_logger<0.6.0>:ale_error_logger_handler:do_log:203]
=========================CRASH REPORT=========================
crasher:
initial call: upr_replicator:init/1
pid: <0.7020.3>
registered_name: 'upr_replicator-AbRegNums-ns_1@172.23.105.160'
exception exit: {{badmatch,
[{<0.25277.9>,
{done,exit,
{normal,
{gen_server,call,
[<0.7021.3>,
{maybe_close_stream,511},
infinity]}},
[{gen_server,call,3,
[{file,"gen_server.erl"},{line,188}]},
{upr_replicator,'
[{file,"src/upr_replicator.erl"},
{line,119}]},
{upr_replicator,'
[{file,"src/upr_replicator.erl"},
{line,189}]}]}}]},
[{misc,sync_shutdown_many_i_am_trapping_exits,1,
[{file,"src/misc.erl"},{line,1475}]},
{upr_replicator,spawn_and_wait,1,
[{file,"src/upr_replicator.erl"},{line,211}]},
{upr_replicator,handle_call,3,
[{file,"src/upr_replicator.erl"},{line,118}]},
{gen_server,handle_msg,5,
[{file,"gen_server.erl"},{line,585}
]},
{proc_lib,init_p_do_apply,3,
[
in function gen_server:terminate/6 (gen_server.erl, line 744)
ancestors: ['upr_sup-AbRegNums','single_bucket_sup-AbRegNums',
<0.6928.3>]
messages: [{'EXIT',<0.7022.3>,
{{badmatch,
[{<0.25277.9>,
{done,exit,
{normal,
{gen_server,call,
[<0.7021.3>,{maybe_close_stream,511},infinity]}},
[{gen_server,call,3,
[{file,"gen_server.erl"},{line,188}]},
{upr_replicator,'
[{file,"src/upr_replicator.erl"},{line,119}]},
{upr_replicator,'
[{file,"src/upr_replicator.erl"},{line,189}]}]}}]},
[{misc,sync_shutdown_many_i_am_trapping_exits,1,
[{file,"src/misc.erl"},{line,1475}]},
{upr_replicator,spawn_and_wait,1,
[{file,"src/upr_replicator.erl"},{line,211}]},
{upr_replicator,handle_call,3,
[{file,"src/upr_replicator.erl"},{line,118}]},
{gen_server,handle_msg,5,
[{file,"gen_server.erl"},{line,585}]},
{proc_lib,init_p_do_apply,3,
[{file,"proc_lib.erl"}
,
{line,239}]}]}}]links: [<0.6949.3>]
dictionary: []
trap_exit: true
status: running
heap_size: 4185
stack_size: 27
reductions: 29323
neighbours:
[error_logger:error,2014-06-28T13:18:40.603,ns_1@172.23.105.159:error_logger<0.6.0>:ale_error_logger_handler:do_log:203]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'upr_sup-AbRegNums'}
Context: child_terminated
Reason: {{badmatch,
[{<0.25277.9>,
{done,exit,
{normal,
{gen_server,call,
[<0.7021.3>,
{maybe_close_stream,511},
infinity]}},
[{gen_server,call,3,
[{file,"gen_server.erl"},{line,188}]},
{upr_replicator,'
[{file,"src/upr_replicator.erl"},{line,119}]},
{upr_replicator,'
[{file,"src/upr_replicator.erl"},
{line,189}]}]}}]},
[{misc,sync_shutdown_many_i_am_trapping_exits,1,
[{file,"src/misc.erl"},{line,1475}]},
{upr_replicator,spawn_and_wait,1,
[{file,"src/upr_replicator.erl"},{line,211}]},
{upr_replicator,handle_call,3,
[{file,"src/upr_replicator.erl"},{line,118}]},
{gen_server,handle_msg,5,
[{file,"gen_server.erl"},{line,585}]},
{proc_lib,init_p_do_apply,3,
[{file,"proc_lib.erl"},{line,239}
]}]}
Offender: [
,
,
{mfargs,{upr_replicator,start_link,undefined}},
,
,
]
[ns_server:debug,2014-06-28T13:18:40.603,ns_1@172.23.105.159:compaction_daemon<0.721.0>:compaction_daemon:handle_info:456]Looks like vbucket mover inhibiting view compaction for for bucket "AbRegNums" is dead. Canceling inhibition
[ns_server:debug,2014-06-28T13:18:40.604,ns_1@172.23.105.159:<0.25170.9>:ns_pubsub:do_subscribe_link:136]Parent process of subscription
exited with reason {unexpected_exit,
{'EXIT',
<0.25286.9>,
{upr_wait_for_data_move_failed,
"AbRegNums",
510,
'ns_1@172.23.105.160',
['ns_1@172.23.105.159',
'ns_1@172.23.105.207'],
{error,
no_stats_for_this_vbucket}}}}