Details
-
Bug
-
Resolution: Cannot Reproduce
-
Critical
-
6.5.0
-
Build 6.5.0-3084
-
Untriaged
-
Centos 64-bit
-
Unknown
Description
Setup:
C1: '172.23.104.201', '172.23.104.202'
C2 : '172.23.105.44', '172.23.105.45'
C1 (default bucket) <-> C2 (default bucket) bidirectional XDCR
Operation:
Use rest api to swap-rebalance
[remove_node:172.23.105.45] and [add_node:172.23.105.47] at C2 cluster 172.23.105.44
Crash:
./couchbase.log: [<<"Started node add transaction by adding node 'ns_1@172.23.105.47' to nodes_wanted (group: undefined)">>],
|
./couchbase.log: [<<"Node 'ns_1@172.23.105.44' saw that node 'ns_1@172.23.105.47' came up. Tags: []">>],
|
./couchbase.log: [<<"Node 'ns_1@172.23.105.45' saw that node 'ns_1@172.23.105.47' came up. Tags: []">>],
|
./couchbase.log: 'ns_1@172.23.105.47',menelaus_sup,1,
|
./couchbase.log: [<<"Couchbase Server has started on web port 8091 on node 'ns_1@172.23.105.47'. Version: \"6.5.0-3084-enterprise\".">>],
|
./couchbase.log: 'ns_1@172.23.105.47',ns_cluster,3,
|
./couchbase.log: [<<"Node ns_1@172.23.105.47 joined cluster">>],
|
./couchbase.log: 'ns_1@172.23.105.47',memcached_config_mgr,0,
|
./couchbase.log: [<<"Starting rebalance, KeepNodes = ['ns_1@172.23.105.47','ns_1@172.23.105.44'], EjectNodes = ['ns_1@172.23.105.45'], Failed over and being ejected nodes = []; no delta recovery nodes; Operation Id = c59f8a7dc2768067c6eb36c11a7dce56">>],
|
./couchbase.log: 'ns_1@172.23.105.47',ns_memcached,0,
|
./couchbase.log: [<<"Bucket \"default\" loaded on node 'ns_1@172.23.105.47' in 0 seconds.">>],
|
./couchbase.log: 'ns_1@172.23.105.47',ns_log,0,
|
./couchbase.log: 'ns_1@172.23.105.47',ns_log,0,
|
./couchbase.log: [<<"<0.17973.21> exited with {unexpected_exit,\n {'EXIT',<0.18686.21>,\n {{wait_seqno_persisted_failed,\"default\",388,207,\n [{'ns_1@172.23.105.47',\n {'EXIT',\n {{{{case_clause,\n {error,\n {{{badmatch,\n {error,\n {{badmatch,{error,timeout}},\n [{mc_client_binary,cmd_vocal_recv,\n 5,\n [{file,\"src/mc_client_binary.erl\"},\n {line,155}]},\n {mc_client_binary,cmd_vocal,3,\n [{file,\"src/mc_client_binary.erl\"},\n {line,140}]},\n {dcp_commands,open_connection,5,\n [{file,\"src/dcp_commands.erl\"},\n {line,79}]},\n {dcp_proxy,connect,5,\n [{file,\"src/dcp_proxy.erl\"},\n {line,247}]},\n {dcp_proxy,maybe_connect,2,\n [{file,\"src/dcp_proxy.erl\"},\n {line,221}]},\n {dcp_producer_conn,init,2,\n [{file,\n \"src/dcp_producer_conn.erl\"},\n {line,31}]},\n {dcp_proxy,init,1,\n [{file,\"src/dcp_proxy.erl\"},\n {line,59}]},\n {gen_server,init_it,2,\n [{file,\"gen_server.erl\"},\n {line,365}]}]}}},\n [{dcp_replicator,init,1,\n [{file,\"src/dcp_replicator.erl\"},\n {line,50}]},\n {gen_server,init_it,2,\n [{file,\"gen_server.erl\"},{line,365}]},\n {gen_server,init_it,6,\n [{file,\"gen_server.erl\"},{line,333}]},\n {proc_lib,init_p_do_apply,3,\n [{file,\"proc_lib.erl\"},{line,247}]}]},\n {child,undefined,\n {'ns_1@172.23.105.45',\n [del_times,json,set_consumer_name,\n snappy,xattr]},\n {dcp_replicator,start_link,\n ['ns_1@172.23.105.45',\"default\",\n [del_times,json,set_consumer_name,\n snappy,xattr]]},\n temporary,60000,worker,\n [dcp_replicator]}}}},\n [{dcp_sup,start_replicator,2,\n [{file,\"src/dcp_sup.erl\"},{line,57}]},\n {dcp_sup,\n '-manage_replicators/2-lc$^3/1-3-',2,\n [{file,\"src/dcp_sup.erl\"},{line,101}]},\n {dcp_replication_manager,handle_call,3,\n [{file,\"src/dcp_replication_manager.erl\"},\n {line,89}]},\n {gen_server,try_handle_call,4,\n [{file,\"gen_server.erl\"},{line,636}]},\n {gen_server,handle_msg,6,\n [{file,\"gen_server.erl\"},{line,665}]},\n {proc_lib,init_p_do_apply,3,\n [{file,\"proc_lib.erl\"},{line,247}]}]},\n {gen_server,call,\n ['dcp_replication_manager-default',\n {get_replicator_pid,388},\n infinity]}},\n {gen_server,call,\n [{'janitor_agent-default',\n 'ns_1@172.23.105.47'},\n {if_rebalance,<0.32155.20>,\n {wait_seqno_persisted,388,207}},\n infinity]}}}}]},\n [{ns_single_vbucket_mover,\n '-wait_seqno_persisted_many/5-fun-2-',5,\n [{file,\"src/ns_single_vbucket_mover.erl\"},\n {line,510}]},\n {proc_lib,init_p,3,\n [{file,\"proc_lib.erl\"},{line,232}]}]}}}">>],
|
./couchbase.log: [<<"Rebalance exited with reason {mover_crashed,\n {unexpected_exit,\n {'EXIT',<0.18686.21>,\n {{wait_seqno_persisted_failed,\"default\",388,\n 207,\n [{'ns_1@172.23.105.47',\n {'EXIT',\n {{{{case_clause,\n {error,\n {{{badmatch,\n {error,\n {{badmatch,{error,timeout}},\n [{mc_client_binary,\n cmd_vocal_recv,5,\n [{file,\n \"src/mc_client_binary.erl\"},\n {line,155}]},\n {mc_client_binary,cmd_vocal,\n 3,\n [{file,\n \"src/mc_client_binary.erl\"},\n {line,140}]},\n {dcp_commands,\n open_connection,5,\n [{file,\n \"src/dcp_commands.erl\"},\n {line,79}]},\n {dcp_proxy,connect,5,\n [{file,\"src/dcp_proxy.erl\"},\n {line,247}]},\n {dcp_proxy,maybe_connect,2,\n [{file,\"src/dcp_proxy.erl\"},\n {line,221}]},\n {dcp_producer_conn,init,2,\n [{file,\n \"src/dcp_producer_conn.erl\"},\n {line,31}]},\n {dcp_proxy,init,1,\n [{file,\"src/dcp_proxy.erl\"},\n {line,59}]},\n {gen_server,init_it,2,\n [{file,\"gen_server.erl\"},\n {line,365}]}]}}},\n [{dcp_replicator,init,1,\n [{file,\"src/dcp_replicator.erl\"},\n {line,50}]},\n {gen_server,init_it,2,\n [{file,\"gen_server.erl\"},\n {line,365}]},\n {gen_server,init_it,6,\n [{file,\"gen_server.erl\"},\n {line,333}]},\n {proc_lib,init_p_do_apply,3,\n [{file,\"proc_lib.erl\"},\n {line,247}]}]},\n {child,undefined,\n {'ns_1@172.23.105.45',\n [del_times,json,\n set_consumer_name,snappy,xattr]},\n {dcp_replicator,start_link,\n ['ns_1@172.23.105.45',\"default\",\n [del_times,json,\n set_consumer_name,snappy,\n xattr]]},\n temporary,60000,worker,\n [dcp_replicator]}}}},\n [{dcp_sup,start_replicator,2,\n [{file,\"src/dcp_sup.erl\"},\n {line,57}]},\n {dcp_sup,\n '-manage_replicators/2-lc$^3/1-3-',\n 2,\n [{file,\"src/dcp_sup.erl\"},\n {line,101}]},\n {dcp_replication_manager,\n handle_call,3,\n [{file,\n \"src/dcp_replication_manager.erl\"},\n {line,89}]},\n {gen_server,try_handle_call,4,\n [{file,\"gen_server.erl\"},\n {line,636}]},\n {gen_server,handle_msg,6,\n [{file,\"gen_server.erl\"},\n {line,665}]},\n {proc_lib,init_p_do_apply,3,\n [{file,\"proc_lib.erl\"},\n {line,247}]}]},\n {gen_server,call,\n ['dcp_replication_manager-default',\n {get_replicator_pid,388},\n infinity]}},\n {gen_server,call,\n [{'janitor_agent-default',\n 'ns_1@172.23.105.47'},\n {if_rebalance,<0.32155.20>,\n {wait_seqno_persisted,388,207}},\n infinity]}}}}]},\n [{ns_single_vbucket_mover,\n '-wait_seqno_persisted_many/5-fun-2-',5,\n [{file,\"src/ns_single_vbucket_mover.erl\"},\n {line,510}]},\n {proc_lib,init_p,3,\n [{file,\"proc_lib.erl\"},{line,232}]}]}}}}. Operation Id = c59f8a7dc2768067c6eb36c11a7dce56">>],
|
[error_logger:error,2019-04-26T20:57:01.046-07:00,ns_1@172.23.105.44:error_logger<0.32.0>:ale_error_logger_handler:do_log:203]
|
=========================CRASH REPORT=========================
|
crasher:
|
initial call: ns_single_vbucket_mover:mover/6
|
pid: <0.17973.21>
|
registered_name: []
|
exception exit: {unexpected_exit,
|
{'EXIT',<0.18686.21>,
|
{{wait_seqno_persisted_failed,"default",388,207,
|
[{'ns_1@172.23.105.47',
|
{'EXIT',
|
{{{{case_clause,
|
{error,
|
{{{badmatch,
|
{error,
|
{{badmatch,{error,timeout}},
|
[{mc_client_binary,cmd_vocal_recv,5,
|
[{file,"src/mc_client_binary.erl"},
|
{line,155}]},
|
{mc_client_binary,cmd_vocal,3,
|
[{file,"src/mc_client_binary.erl"},
|
{line,140}]},
|
{dcp_commands,open_connection,5,
|
[{file,"src/dcp_commands.erl"},
|
{line,79}]},
|
{dcp_proxy,connect,5,
|
[{file,"src/dcp_proxy.erl"},
|
{line,247}]},
|
{dcp_proxy,maybe_connect,2,
|
[{file,"src/dcp_proxy.erl"},
|
{line,221}]},
|
{dcp_producer_conn,init,2,
|
[{file,"src/dcp_producer_conn.erl"},
|
{line,31}]},
|
{dcp_proxy,init,1,
|
[{file,"src/dcp_proxy.erl"},{line,59}]},
|
{gen_server,init_it,2,
|
[{file,"gen_server.erl"},
|
{line,365}]}]}}},
|
[{dcp_replicator,init,1,
|
[{file,"src/dcp_replicator.erl"},
|
{line,50}]},
|
{gen_server,init_it,2,
|
[{file,"gen_server.erl"},{line,365}]},
|
{gen_server,init_it,6,
|
[{file,"gen_server.erl"},{line,333}]},
|
{proc_lib,init_p_do_apply,3,
|
[{file,"proc_lib.erl"},{line,247}]}]},
|
{child,undefined,
|
{'ns_1@172.23.105.45',
|
[del_times,json,set_consumer_name,snappy,
|
xattr]},
|
{dcp_replicator,start_link,
|
['ns_1@172.23.105.45',"default",
|
[del_times,json,set_consumer_name,snappy,
|
xattr]]},
|
temporary,60000,worker,
|
[dcp_replicator]}}}},
|
[{dcp_sup,start_replicator,2,
|
[{file,"src/dcp_sup.erl"},{line,57}]},
|
{dcp_sup,'-manage_replicators/2-lc$^3/1-3-',2,
|
[{file,"src/dcp_sup.erl"},{line,101}]},
|
{dcp_replication_manager,handle_call,3,
|
[{file,"src/dcp_replication_manager.erl"},
|
{line,89}]},
|
{gen_server,try_handle_call,4,
|
[{file,"gen_server.erl"},{line,636}]},
|
{gen_server,handle_msg,6,
|
[{file,"gen_server.erl"},{line,665}]},
|
{proc_lib,init_p_do_apply,3,
|
[{file,"proc_lib.erl"},{line,247}]}]},
|
{gen_server,call,
|
['dcp_replication_manager-default',
|
{get_replicator_pid,388},
|
infinity]}},
|
{gen_server,call,
|
[{'janitor_agent-default','ns_1@172.23.105.47'},
|
{if_rebalance,<0.32155.20>,
|
{wait_seqno_persisted,388,207}},
|
infinity]}}}}]},
|
[{ns_single_vbucket_mover,
|
'-wait_seqno_persisted_many/5-fun-2-',5,
|
[{file,"src/ns_single_vbucket_mover.erl"},
|
{line,510}]},
|
{proc_lib,init_p,3,
|
[{file,"proc_lib.erl"},{line,232}]}]}}}
|
in function ns_single_vbucket_mover:spawn_and_wait/1 (src/ns_single_vbucket_mover.erl, line 109)
|
in call from ns_single_vbucket_mover:wait_master_seqno_persisted_on_replicas/5 (src/ns_single_vbucket_mover.erl, line 495)
|
in call from ns_single_vbucket_mover:'-mover/6-fun-3-'/7 (src/ns_single_vbucket_mover.erl, line 74)
|
in call from misc:try_with_maybe_ignorant_after/2 (src/misc.erl, line 1330)
|
in call from ns_single_vbucket_mover:mover/6 (src/ns_single_vbucket_mover.erl, line 71)
|
ancestors: [<0.32155.20>,<0.31886.20>]
|
message_queue_len: 0
|
messages: []
|
links: [<0.32155.20>]
|
dictionary: [{cleanup_list,[<0.18686.21>]}]
|
trap_exit: true
|
status: running
|
heap_size: 6772
|
stack_size: 27
|
reductions: 58346
|
neighbours:
|