Details
-
Technical task
-
Resolution: Fixed
-
Major
-
2.0-beta-2, 2.0
-
Security Level: Public
-
None
-
windows server r2 2008
build 1950
<manifest><remote name="couchbase" fetch="git://10.1.1.210/"/><remote name="membase" fetch="git://10.1.1.210/"/><remote name="apache" fetch="git://github.com/apache/"/><remote name="erlang" fetch="git://github.com/erlang/"/><default remote="couchbase" revision="master"/><project name="tlm" path="tlm" revision="12abea946eafd7411273d18a10ae1f84390db3d4"><copyfile src="Makefile.top" dest="Makefile"/></project><project name="bucket_engine" path="bucket_engine" revision="70b3624abc697b7d18bf3d57f331b7674544e1e7"/><project name="ep-engine" path="ep-engine" revision="a8db69fa759d333246e690c52ef9497c858012ef"/><project name="libconflate" path="libconflate" revision="2cc8eff8e77d497d9f03a30fafaecb85280535d6"/><project name="libmemcached" path="libmemcached" revision="ca739a890349ac36dc79447e37da7caa9ae819f5" remote="membase"/><project name="libvbucket" path="libvbucket" revision="00d3763593c116e8e5d97aa0b646c42885727398"/><project name="membase-cli" path="membase-cli" revision="f75b10e3a11e41b28568dde028a4284efdab084d" remote="membase"/><project name="memcached" path="memcached" revision="7ea975a93a0231393502af4ca98976eee8a83386" remote="membase"/><project name="moxi" path="moxi" revision="52a5fa887bfff0bf719c4ee5f29634dd8707500e"/><project name="ns_server" path="ns_server" revision="56a7e106905000cbbd8d9a20e77d0bfe9c1d2fda"/><project name="portsigar" path="portsigar" revision="1bc865e1622fb93a3fe0d1a4cdf18eb97ed9d600"/><project name="sigar" path="sigar" revision="63a3cd1b316d2d4aa6dd31ce8fc66101b983e0b0"/><project name="couchbase-examples" path="couchbase-examples" revision="c59551adf11860963c1bba028adf33529a4a4d4a"/><project name="couchbase-python-client" path="couchbase-python-client" revision="006c1aa8b76f6bce11109af8a309133b57079c4c"/><project name="couchdb" path="couchdb" revision="00105ea6d44c15aefcf514a9a2c65fbbb6151830"/><project name="couchdbx-app" path="couchdbx-app" revision="a1cbfcaa0091b4b7038bbe949b9d48787cc526eb"/><project name="couchstore" path="couchstore" revision="b5937c4479bf05dcc67264efe19abaf52870a127"/><project name="geocouch" path="geocouch" revision="849d5443689b1924f097548af864c539bffcc929"/><project name="mccouch" path="mccouch" revision="88701cc326bc3dde4ed072bb8441be83adcfb2a5"/><project name="testrunner" path="testrunner" revision="08155ea827ea0508f47406b36e7c892d8591abf9"/><project name="otp" path="otp" revision="b6dc1a844eab061d0a7153d46e7e68296f15a504" remote="erlang"/><project name="icu4c" path="icu4c" revision="26359393672c378f41f2103a8699c4357c894be7" remote="couchbase"/><project name="snappy" path="snappy" revision="5681dde156e9d07adbeeab79666c9a9d7a10ec95" remote="couchbase"/><project name="v8" path="v8" revision="447decb75060a106131ab4de934bcc374648e7f2" remote="couchbase"/><project name="gperftools" path="gperftools" revision="8f60ba949fb8576c530ef4be148bff97106ddc59" remote="couchbase"/><project name="pysqlite" path="pysqlite" revision="0ff6e32ea05037fddef1eb41a648f2a2141009ea" remote="couchbase"/></manifest>windows server r2 2008 build 1950 <manifest><remote name="couchbase" fetch=" git://10.1.1.210/ "/><remote name="membase" fetch=" git://10.1.1.210/ "/><remote name="apache" fetch=" git://github.com/apache/ "/><remote name="erlang" fetch=" git://github.com/erlang/ "/><default remote="couchbase" revision="master"/><project name="tlm" path="tlm" revision="12abea946eafd7411273d18a10ae1f84390db3d4"><copyfile src="Makefile.top" dest="Makefile"/></project><project name="bucket_engine" path="bucket_engine" revision="70b3624abc697b7d18bf3d57f331b7674544e1e7"/><project name="ep-engine" path="ep-engine" revision="a8db69fa759d333246e690c52ef9497c858012ef"/><project name="libconflate" path="libconflate" revision="2cc8eff8e77d497d9f03a30fafaecb85280535d6"/><project name="libmemcached" path="libmemcached" revision="ca739a890349ac36dc79447e37da7caa9ae819f5" remote="membase"/><project name="libvbucket" path="libvbucket" revision="00d3763593c116e8e5d97aa0b646c42885727398"/><project name="membase-cli" path="membase-cli" revision="f75b10e3a11e41b28568dde028a4284efdab084d" remote="membase"/><project name="memcached" path="memcached" revision="7ea975a93a0231393502af4ca98976eee8a83386" remote="membase"/><project name="moxi" path="moxi" revision="52a5fa887bfff0bf719c4ee5f29634dd8707500e"/><project name="ns_server" path="ns_server" revision="56a7e106905000cbbd8d9a20e77d0bfe9c1d2fda"/><project name="portsigar" path="portsigar" revision="1bc865e1622fb93a3fe0d1a4cdf18eb97ed9d600"/><project name="sigar" path="sigar" revision="63a3cd1b316d2d4aa6dd31ce8fc66101b983e0b0"/><project name="couchbase-examples" path="couchbase-examples" revision="c59551adf11860963c1bba028adf33529a4a4d4a"/><project name="couchbase-python-client" path="couchbase-python-client" revision="006c1aa8b76f6bce11109af8a309133b57079c4c"/><project name="couchdb" path="couchdb" revision="00105ea6d44c15aefcf514a9a2c65fbbb6151830"/><project name="couchdbx-app" path="couchdbx-app" revision="a1cbfcaa0091b4b7038bbe949b9d48787cc526eb"/><project name="couchstore" path="couchstore" revision="b5937c4479bf05dcc67264efe19abaf52870a127"/><project name="geocouch" path="geocouch" revision="849d5443689b1924f097548af864c539bffcc929"/><project name="mccouch" path="mccouch" revision="88701cc326bc3dde4ed072bb8441be83adcfb2a5"/><project name="testrunner" path="testrunner" revision="08155ea827ea0508f47406b36e7c892d8591abf9"/><project name="otp" path="otp" revision="b6dc1a844eab061d0a7153d46e7e68296f15a504" remote="erlang"/><project name="icu4c" path="icu4c" revision="26359393672c378f41f2103a8699c4357c894be7" remote="couchbase"/><project name="snappy" path="snappy" revision="5681dde156e9d07adbeeab79666c9a9d7a10ec95" remote="couchbase"/><project name="v8" path="v8" revision="447decb75060a106131ab4de934bcc374648e7f2" remote="couchbase"/><project name="gperftools" path="gperftools" revision="8f60ba949fb8576c530ef4be148bff97106ddc59" remote="couchbase"/><project name="pysqlite" path="pysqlite" revision="0ff6e32ea05037fddef1eb41a648f2a2141009ea" remote="couchbase"/></manifest>
Description
rebalance in 5->7 node is failed:
Control connection to memcached on 'ns_1@10.3.2.141' disconnected: {badmatch,{error,closed}}
....
[ns_server:info,2012-11-14T20:44:36.251,ns_1@10.3.2.131:ns_port_memcached<0.437.0>:ns_port_server:log:171]memcached<0.437.0>: Wed Nov 14 20:44:35.989437 Pacific Standard Time 3: TAP (Producer) eq_tapq:replication_building_676_'ns_1@10.3.2.130' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 676
memcached<0.437.0>: Wed Nov 14 20:44:36.083187 Pacific Standard Time 3: Notified the completion of checkpoint persistence for vbucket 404, cookie 0000000005A0EB00
[rebalance:info,2012-11-14T20:44:36.345,ns_1@10.3.2.131:<0.7540.93>:ebucketmigrator_srv:init:551]Starting tap stream:
[
{checkpoints,[{953,3}]},
{name,<<"replication_building_953_'ns_1@10.3.2.131'">>},
{takeover,false}]
{{"10.3.2.130",11209},
{"10.3.2.131",11209},
[{vbuckets,[953]}
,
,
,
,
]}
[error_logger:error,2012-11-14T20:44:37.876,ns_1@10.3.2.131:error_logger<0.5.0>:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_single_vbucket_mover:mover/6
pid: <0.7325.93>
registered_name: []
exception exit: {unexpected_exit,
{'EXIT',<0.7456.93>,
{{wait_checkpoint_persisted_failed,"default",404,4,
[{'ns_1@10.3.2.141',
{'EXIT',
{badmatch,{error,closed,
{gen_server,call,
['ns_memcached-default',
,
infinity]}},
{gen_server,call,
[
,
{if_rebalance,<0.27487.91>,
{wait_checkpoint_persisted,404,4}},
infinity]}}}}]},
[
]}}}
in function ns_single_vbucket_mover:spawn_and_wait/1
in call from ns_single_vbucket_mover:mover_inner/6
in call from misc:try_with_maybe_ignorant_after/2
in call from ns_single_vbucket_mover:mover/6
ancestors: [<0.27487.91>,<0.27357.91>]
messages: [{'EXIT',<0.27487.91>,{mover_failed,downstream_closed}}]
links: [<0.27487.91>]
dictionary: [
]
trap_exit: true
status: running
heap_size: 6765
stack_size: 24
reductions: 12605
neighbours:
...
[error_logger:error,2012-11-14T20:44:37.938,ns_1@10.3.2.131:error_logger<0.5.0>:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ebucketmigrator_srv:confirm_sent_messages/1-fun-0/0
pid: <20523.8371.14>
registered_name: []
exception error: no match of right hand side value
in function ebucketmigrator_srv:'confirm_sent_messages/1-fun-0'/3
ancestors: [<20523.8294.14>,<0.7410.93>,<0.7325.93>,<0.27487.91>,
<0.27357.91>]
messages: []
links: [<20523.8294.14>]
dictionary: []
trap_exit: false
status: running
heap_size: 987
stack_size: 24
reductions: 748
neighbours:
[error_logger:error,2012-11-14T20:44:37.954,ns_1@10.3.2.131:error_logger<0.5.0>:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ebucketmigrator_srv:init/1
pid: <20523.8294.14>
registered_name: []
exception exit: downstream_closed
in function gen_server:terminate/6
ancestors: [<0.7410.93>,<0.7325.93>,<0.27487.91>,<0.27357.91>]
messages: [
]
links: [#Port<20523.253270>,<20523.8371.14>,<0.7410.93>,
#Port<20523.253269>]
dictionary: []
trap_exit: true
status: running
heap_size: 1597
stack_size: 24
reductions: 84633
neighbours:
...
[ns_server:error,2012-11-14T20:44:42.720,ns_1@10.3.2.131:<0.7424.93>:misc:inner_wait_shutdown:1426]Expected exit signal from <0.7426.93> but could not get it in 5 seconds. This is a bug, but process we're waiting for is dead (noproc), so trying to ignore...
[ns_server:error,2012-11-14T20:44:42.735,ns_1@10.3.2.131:<0.7424.93>:misc:sync_shutdown_many_i_am_trapping_exits:1408]Shutdown of the following failed: [
{<0.7426.93>,noproc},
{<20581.12222.6>,normal}]
[ns_server:info,2012-11-14T20:44:42.782,ns_1@10.3.2.131:<0.7424.93>:ns_replicas_builder_utils:kill_a_bunch_of_tap_names:59]Killed the following tap names on 'ns_1@10.3.2.141': [<<"replication_building_128_'ns_1@10.3.2.139'">>,
<<"replication_building_128_'ns_1@10.3.2.131'">>,
<<"replication_building_128_'ns_1@10.3.2.130'">>,
<<"replication_building_128_'ns_1@10.3.2.132'">>]
[error_logger:error,2012-11-14T20:44:42.782,ns_1@10.3.2.131:error_logger<0.5.0>:ale_error_logger_handler:log_msg:76]** Generic server <0.7424.93> terminating
** Last message in was {'EXIT',<0.7426.93>,normal}
** When Server state == {state,"default",128,'ns_1@10.3.2.141',
[{'ns_1@10.3.2.139',<20200.30113.21>},
{'ns_1@10.3.2.131',<0.7426.93>},
{'ns_1@10.3.2.130',<20199.10253.29>},
{'ns_1@10.3.2.132',<20581.12222.6>}]}
** Reason for termination ==
** {{badmatch,[{<20200.30113.21>,normal}
,
{<20581.12222.6>,normal}]},
[{misc,sync_shutdown_many_i_am_trapping_exits,1},
{misc,try_with_maybe_ignorant_after,2},
{gen_server,terminate,6},
{proc_lib,init_p_do_apply,3}]}
[ns_server:error,2012-11-14T20:44:42.782,ns_1@10.3.2.131:<0.7368.93>:misc:sync_shutdown_many_i_am_trapping_exits:1408]Shutdown of the following failed: [{<0.7424.93>,
{{badmatch,
[{<20200.30113.21>,normal},
{<0.7426.93>,noproc}
,
[{misc, sync_shutdown_many_i_am_trapping_exits, 1},
{misc,try_with_maybe_ignorant_after,2},
{gen_server,terminate,6},
{proc_lib,init_p_do_apply,3}]}},
{<0.7429.93>,
{{badmatch,
[{'EXIT',
{normal,
{gen_server,call, [<0.7426.93>,had_backfill,30000]}}},
{'EXIT',
{shutdown,
{gen_server,call, [<20199.10253.29>,had_backfill, 30000]}}}]},
[{ns_single_vbucket_mover, '-wait_backfill_determination/1-fun-1-', 1}]}}]
[ns_server:error,2012-11-14T20:44:42.782,ns_1@10.3.2.131:<0.7368.93>:misc:try_with_maybe_ignorant_after:1444]Eating exception from ignorant after-block:
{error,
{badmatch,
[{<0.7424.93>,
{{badmatch,
[{<20200.30113.21>,normal},
{<0.7426.93>,noproc},
{<20581.12222.6>,normal}
]},
[
{misc,try_with_maybe_ignorant_after,2},
{gen_server,terminate,6},
{proc_lib,init_p_do_apply,3}]}},
{<0.7429.93>,
{{badmatch,
[{'EXIT',
{normal,
{gen_server,call,[<0.7426.93>,had_backfill,30000]}}},
{'EXIT',
{shutdown,
{gen_server,call, [<20199.10253.29>,had_backfill,30000]}}}]},
[{ns_single_vbucket_mover,'-wait_backfill_determination/1-fun-1-', 1}]}}]},
[{misc,sync_shutdown_many_i_am_trapping_exits,1}
,
{ns_single_vbucket_mover,mover,6},
{proc_lib,init_p_do_apply,3}]}
[error_logger:error,2012-11-14T20:44:42.782,ns_1@10.3.2.131:error_logger<0.5.0>:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: new_ns_replicas_builder:init/1
pid: <0.7424.93>
registered_name: []
exception exit: {{badmatch,[{<20200.30113.21>,normal},
{<0.7426.93>,noproc},
{<20581.12222.6>,normal}]},
[{misc,sync_shutdown_many_i_am_trapping_exits,1},
{misc,try_with_maybe_ignorant_after,2}
,
,
]}
in function gen_server:terminate/6
ancestors: [<0.7368.93>,<0.27487.91>,<0.27357.91>]
messages: [
]
links: [<0.7368.93>]
dictionary: []
trap_exit: true
status: running
heap_size: 514229
stack_size: 24
reductions: 66044
neighbours:
[user:info,2012-11-14T20:44:42.798,ns_1@10.3.2.131:<0.385.0>:ns_orchestrator:handle_info:319]Rebalance exited with reason
{mover_failed,downstream_closed}