Details
-
Bug
-
Resolution: Duplicate
-
Critical
-
2.0
-
None
-
Security Level: Public
-
None
-
build 1954
<manifest>
<remote name="couchbase" fetch="git://10.1.1.210/"/>
<remote name="membase" fetch="git://10.1.1.210/"/>
<remote name="apache" fetch="git://github.com/apache/"/>
<remote name="erlang" fetch="git://github.com/erlang/"/>
<default remote="couchbase" revision="master"/>
<project name="tlm" path="tlm" revision="12abea946eafd7411273d18a10ae1f84390db3d4">
<copyfile src="Makefile.top" dest="Makefile"/>
</project>
<project name="bucket_engine" path="bucket_engine" revision="70b3624abc697b7d18bf3d57f331b7674544e1e7"/>
<project name="ep-engine" path="ep-engine" revision="9b8377b94356c867cab2f63b2b5798001ca671f7"/>
<project name="libconflate" path="libconflate" revision="2cc8eff8e77d497d9f03a30fafaecb85280535d6"/>
<project name="libmemcached" path="libmemcached" revision="ca739a890349ac36dc79447e37da7caa9ae819f5" remote="membase"/>
<project name="libvbucket" path="libvbucket" revision="00d3763593c116e8e5d97aa0b646c42885727398"/>
<project name="membase-cli" path="membase-cli" revision="9daab001105fc00ed00550811a3b3725ef17f19e" remote="membase"/>
<project name="memcached" path="memcached" revision="7ea975a93a0231393502af4ca98976eee8a83386" remote="membase"/>
<project name="moxi" path="moxi" revision="52a5fa887bfff0bf719c4ee5f29634dd8707500e"/>
<project name="ns_server" path="ns_server" revision="1f94354464dcd96414a33e74e699a2bae9cecee2"/>
<project name="portsigar" path="portsigar" revision="1bc865e1622fb93a3fe0d1a4cdf18eb97ed9d600"/>
<project name="sigar" path="sigar" revision="63a3cd1b316d2d4aa6dd31ce8fc66101b983e0b0"/>
<project name="couchbase-examples" path="couchbase-examples" revision="c59551adf11860963c1bba028adf33529a4a4d4a"/>
<project name="couchbase-python-client" path="couchbase-python-client" revision="006c1aa8b76f6bce11109af8a309133b57079c4c"/>
<project name="couchdb" path="couchdb" revision="31560d74c3bbe8c019186923a9db3468a8197ab8"/>
<project name="couchdbx-app" path="couchdbx-app" revision="7b8eb60114fdc3b8e79baf8cc8ec87d32be71983"/>
<project name="couchstore" path="couchstore" revision="b5937c4479bf05dcc67264efe19abaf52870a127"/>
<project name="geocouch" path="geocouch" revision="849d5443689b1924f097548af864c539bffcc929"/>
<project name="mccouch" path="mccouch" revision="88701cc326bc3dde4ed072bb8441be83adcfb2a5"/>
<project name="testrunner" path="testrunner" revision="23c4acee1142b71d91aeadef44719ec55b8390c0"/>
<project name="otp" path="otp" revision="b6dc1a844eab061d0a7153d46e7e68296f15a504" remote="erlang"/>
<project name="icu4c" path="icu4c" revision="26359393672c378f41f2103a8699c4357c894be7" remote="couchbase"/>
<project name="snappy" path="snappy" revision="5681dde156e9d07adbeeab79666c9a9d7a10ec95" remote="couchbase"/>
<project name="v8" path="v8" revision="447decb75060a106131ab4de934bcc374648e7f2" remote="couchbase"/>
<project name="gperftools" path="gperftools" revision="8f60ba949fb8576c530ef4be148bff97106ddc59" remote="couchbase"/>
<project name="pysqlite" path="pysqlite" revision="0ff6e32ea05037fddef1eb41a648f2a2141009ea" remote="couchbase"/>
</manifest>build 1954 <manifest> <remote name="couchbase" fetch=" git://10.1.1.210/ "/> <remote name="membase" fetch=" git://10.1.1.210/ "/> <remote name="apache" fetch=" git://github.com/apache/ "/> <remote name="erlang" fetch=" git://github.com/erlang/ "/> <default remote="couchbase" revision="master"/> <project name="tlm" path="tlm" revision="12abea946eafd7411273d18a10ae1f84390db3d4"> <copyfile src="Makefile.top" dest="Makefile"/> </project> <project name="bucket_engine" path="bucket_engine" revision="70b3624abc697b7d18bf3d57f331b7674544e1e7"/> <project name="ep-engine" path="ep-engine" revision="9b8377b94356c867cab2f63b2b5798001ca671f7"/> <project name="libconflate" path="libconflate" revision="2cc8eff8e77d497d9f03a30fafaecb85280535d6"/> <project name="libmemcached" path="libmemcached" revision="ca739a890349ac36dc79447e37da7caa9ae819f5" remote="membase"/> <project name="libvbucket" path="libvbucket" revision="00d3763593c116e8e5d97aa0b646c42885727398"/> <project name="membase-cli" path="membase-cli" revision="9daab001105fc00ed00550811a3b3725ef17f19e" remote="membase"/> <project name="memcached" path="memcached" revision="7ea975a93a0231393502af4ca98976eee8a83386" remote="membase"/> <project name="moxi" path="moxi" revision="52a5fa887bfff0bf719c4ee5f29634dd8707500e"/> <project name="ns_server" path="ns_server" revision="1f94354464dcd96414a33e74e699a2bae9cecee2"/> <project name="portsigar" path="portsigar" revision="1bc865e1622fb93a3fe0d1a4cdf18eb97ed9d600"/> <project name="sigar" path="sigar" revision="63a3cd1b316d2d4aa6dd31ce8fc66101b983e0b0"/> <project name="couchbase-examples" path="couchbase-examples" revision="c59551adf11860963c1bba028adf33529a4a4d4a"/> <project name="couchbase-python-client" path="couchbase-python-client" revision="006c1aa8b76f6bce11109af8a309133b57079c4c"/> <project name="couchdb" path="couchdb" revision="31560d74c3bbe8c019186923a9db3468a8197ab8"/> <project name="couchdbx-app" path="couchdbx-app" revision="7b8eb60114fdc3b8e79baf8cc8ec87d32be71983"/> <project name="couchstore" path="couchstore" revision="b5937c4479bf05dcc67264efe19abaf52870a127"/> <project name="geocouch" path="geocouch" revision="849d5443689b1924f097548af864c539bffcc929"/> <project name="mccouch" path="mccouch" revision="88701cc326bc3dde4ed072bb8441be83adcfb2a5"/> <project name="testrunner" path="testrunner" revision="23c4acee1142b71d91aeadef44719ec55b8390c0"/> <project name="otp" path="otp" revision="b6dc1a844eab061d0a7153d46e7e68296f15a504" remote="erlang"/> <project name="icu4c" path="icu4c" revision="26359393672c378f41f2103a8699c4357c894be7" remote="couchbase"/> <project name="snappy" path="snappy" revision="5681dde156e9d07adbeeab79666c9a9d7a10ec95" remote="couchbase"/> <project name="v8" path="v8" revision="447decb75060a106131ab4de934bcc374648e7f2" remote="couchbase"/> <project name="gperftools" path="gperftools" revision="8f60ba949fb8576c530ef4be148bff97106ddc59" remote="couchbase"/> <project name="pysqlite" path="pysqlite" revision="0ff6e32ea05037fddef1eb41a648f2a2141009ea" remote="couchbase"/> </manifest>
Description
for reproduce use -t rebalancetests.IncrementalRebalanceInTests.test_load,replica=2,delete-ratio=0.6,expiry-ratio=0.2,do-stop=True
cluster was incrementally rebalanced from 1 to 5 nodes. Rebalance 4->5 nodes is failed. Seeing error to persist DELETE keys:
2012-11-21 04:34:03.664 ns_orchestrator:4:info:message(ns_1@10.1.3.82) - Starting rebalance, KeepNodes = ['ns_1@10.1.3.83','ns_1@10.1.3.86',
'ns_1@10.1.3.84','ns_1@10.1.3.85',
'ns_1@10.1.3.82'], EjectNodes = []
...
2012-11-21 04:38:31.133 menelaus_web_alerts_srv:1:info:message(ns_1@10.1.3.82) - Write Commit Failure. Disk write failed for item in Bucket "bucket-0" on node 10.1.3.82.
2012-11-21 04:38:36.758 ns_port_server:0:info:message(ns_1@10.1.3.82) - Port server memcached on node 'ns_1@10.1.3.82' exited with status 255. Restarting. Messages: Wed Nov 21 04:38:28.556445 Pacific Standard Time 3: Fatal error in persisting DELETE ``35394-8cb6bb7'' on vb 758!!! Requeue it...
Wed Nov 21 04:38:28.556445 Pacific Standard Time 3: Fatal error in persisting DELETE ``35937-8cb6bb7'' on vb 758!!! Requeue it...
Wed Nov 21 04:38:28.556445 Pacific Standard Time 3: Fatal error in persisting DELETE ``35989-ed566c9'' on vb 758!!! Requeue it...
Wed Nov 21 04:38:28.556445 Pacific Standard Time 3: Fatal error in persisting DELETE ``37231-8cb6bb7'' on vb 758!!! Requeue it...
Wed Nov 21 04:38:28.556445 Pacific Standard Time 3: Fatal error in persisting DELETE ``3736-ed566c9'' on vb 758!!! Requeue it...
Wed Nov 21 04:38:28.556445 Pacific Standard Time 3: Fatal error in persisting DELETE ``3788-8cb6bb7'' on vb 758!!! Requeue it...
Wed Nov 21 04:38:28.556445 Pacific Standard Time 3: Fatal error in persisting DELETE ``37892-8cb6bb7'' on vb 758!!! Requeue it...
Wed Nov 21 04:38:28.556445 Pacific Standard Time 3: Fatal error in persisting DELETE ``38337-8cb6bb7'' on vb 758!!! Requeue it...
Wed Nov 21 04:38:28.556445 Pacific Standard Time 3: Fatal error in persisting DELETE ``38994-8cb6bb7'' on vb 758!!! Requeue it...
Wed Nov 21 04:38:28.556445 Pacific Standard Time 3: Fatal error in persisting DELETE ``39772-8cb6bb7'' on vb 758!!! Requeue it...
...
Wed Nov 21 04:38:36.275195 Pacific Standard Time 3: TAP (Producer) eq_tapq:replication_ns_1@10.1.3.84 - Sending TAP_OPAQUE with command "complete_vb_filter_change" and vbucket 0
...
2012-11-21 04:38:36.836 ns_memcached:4:info:message(ns_1@10.1.3.82) - Control connection to memcached on 'ns_1@10.1.3.82' disconnected: {badmatch,
{error,
closed}}
2012-11-21 04:38:42.586 ns_vbucket_mover:0:critical:message(ns_1@10.1.3.82) - <0.18589.34> exited with {unexpected_exit,
{'EXIT',<0.18653.34>,
{{wait_checkpoint_persisted_failed,"bucket-0",759,
6,
[{'ns_1@10.1.3.82',
{'EXIT',
{badmatch,{error,closed,
{gen_server,call,
['ns_memcached-bucket-0',
,
infinity]}},
{gen_server,call,
[
,
{if_rebalance,<0.22501.33>,
{wait_checkpoint_persisted,759,6}},
infinity]}}}}]},
[
]}}}
attaching logs