Details
-
Bug
-
Resolution: Cannot Reproduce
-
Major
-
None
-
Security Level: Public
-
None
-
build 1862
windows server r2 2008
<manifest><remote name="couchbase" fetch="git://10.1.1.210/"/><remote name="membase" fetch="git://10.1.1.210/"/><remote name="apache" fetch="git://github.com/apache/"/><remote name="erlang" fetch="git://github.com/erlang/"/><default remote="couchbase" revision="master"/><project name="tlm" path="tlm" revision="ab70f6d42f46621ec576889e57cb37ac2d64a84b"><copyfile dest="Makefile" src="Makefile.top"/></project><project name="bucket_engine" path="bucket_engine" revision="70b3624abc697b7d18bf3d57f331b7674544e1e7"/><project name="ep-engine" path="ep-engine" revision="8765e790c2da66e076b5ff51ae3fc161dc0fab01"/><project name="libconflate" path="libconflate" revision="2cc8eff8e77d497d9f03a30fafaecb85280535d6"/><project name="libmemcached" path="libmemcached" revision="ca739a890349ac36dc79447e37da7caa9ae819f5" remote="membase"/><project name="libvbucket" path="libvbucket" revision="00d3763593c116e8e5d97aa0b646c42885727398"/><project name="membase-cli" path="membase-cli" revision="c82db287eab652d25116b042d4627a6931722a8e" remote="membase"/><project name="memcached" path="memcached" revision="06ab906e6702917c4b6b90a6b0051644719a357d" remote="membase"/><project name="moxi" path="moxi" revision="52a5fa887bfff0bf719c4ee5f29634dd8707500e"/><project name="ns_server" path="ns_server" revision="a8f7103eff84829a01ebe2a7ec8d0b5c91b2a161"/><project name="portsigar" path="portsigar" revision="1bc865e1622fb93a3fe0d1a4cdf18eb97ed9d600"/><project name="sigar" path="sigar" revision="63a3cd1b316d2d4aa6dd31ce8fc66101b983e0b0"/><project name="couchbase-examples" path="couchbase-examples" revision="21e6161a1d064979b5c6aa99cd34ccc41c9d7aca"/><project name="couchbase-python-client" path="couchbase-python-client" revision="86b398e4fbc1f2e38d356e14df0c1bb4e3d2427b"/><project name="couchdb" path="couchdb" revision="69032c6dbcd64c056265416d2c49b1a9ee06f9c3"/><project name="couchdbx-app" path="couchdbx-app" revision="76d79be79c1454cff0f878d5a88a792270ec1b17"/><project name="couchstore" path="couchstore" revision="772f66a29a81be59d8bdaaa74b3898bb44fcc7e2"/><project name="geocouch" path="geocouch" revision="b0bd742551639c52030c070e5bf9390edbb536ba"/><project name="mccouch" path="mccouch" revision="88701cc326bc3dde4ed072bb8441be83adcfb2a5"/><project name="testrunner" path="testrunner" revision="f7a0a23b32c73866444b3a795fad647c350f47d9"/><project name="otp" path="otp" revision="b6dc1a844eab061d0a7153d46e7e68296f15a504" remote="erlang"/><project name="icu4c" path="icu4c" revision="26359393672c378f41f2103a8699c4357c894be7" remote="couchbase"/><project name="snappy" path="snappy" revision="5681dde156e9d07adbeeab79666c9a9d7a10ec95" remote="couchbase"/><project name="v8" path="v8" revision="447decb75060a106131ab4de934bcc374648e7f2" remote="couchbase"/><project name="gperftools" path="gperftools" revision="8f60ba949fb8576c530ef4be148bff97106ddc59" remote="couchbase"/><project name="pysqlite" path="pysqlite" revision="0ff6e32ea05037fddef1eb41a648f2a2141009ea" remote="couchbase"/></manifest>
build 1862 windows server r2 2008 <manifest><remote name="couchbase" fetch=" git://10.1.1.210/ "/><remote name="membase" fetch=" git://10.1.1.210/ "/><remote name="apache" fetch=" git://github.com/apache/ "/><remote name="erlang" fetch=" git://github.com/erlang/ "/><default remote="couchbase" revision="master"/><project name="tlm" path="tlm" revision="ab70f6d42f46621ec576889e57cb37ac2d64a84b"><copyfile dest="Makefile" src="Makefile.top"/></project><project name="bucket_engine" path="bucket_engine" revision="70b3624abc697b7d18bf3d57f331b7674544e1e7"/><project name="ep-engine" path="ep-engine" revision="8765e790c2da66e076b5ff51ae3fc161dc0fab01"/><project name="libconflate" path="libconflate" revision="2cc8eff8e77d497d9f03a30fafaecb85280535d6"/><project name="libmemcached" path="libmemcached" revision="ca739a890349ac36dc79447e37da7caa9ae819f5" remote="membase"/><project name="libvbucket" path="libvbucket" revision="00d3763593c116e8e5d97aa0b646c42885727398"/><project name="membase-cli" path="membase-cli" revision="c82db287eab652d25116b042d4627a6931722a8e" remote="membase"/><project name="memcached" path="memcached" revision="06ab906e6702917c4b6b90a6b0051644719a357d" remote="membase"/><project name="moxi" path="moxi" revision="52a5fa887bfff0bf719c4ee5f29634dd8707500e"/><project name="ns_server" path="ns_server" revision="a8f7103eff84829a01ebe2a7ec8d0b5c91b2a161"/><project name="portsigar" path="portsigar" revision="1bc865e1622fb93a3fe0d1a4cdf18eb97ed9d600"/><project name="sigar" path="sigar" revision="63a3cd1b316d2d4aa6dd31ce8fc66101b983e0b0"/><project name="couchbase-examples" path="couchbase-examples" revision="21e6161a1d064979b5c6aa99cd34ccc41c9d7aca"/><project name="couchbase-python-client" path="couchbase-python-client" revision="86b398e4fbc1f2e38d356e14df0c1bb4e3d2427b"/><project name="couchdb" path="couchdb" revision="69032c6dbcd64c056265416d2c49b1a9ee06f9c3"/><project name="couchdbx-app" path="couchdbx-app" revision="76d79be79c1454cff0f878d5a88a792270ec1b17"/><project name="couchstore" path="couchstore" revision="772f66a29a81be59d8bdaaa74b3898bb44fcc7e2"/><project name="geocouch" path="geocouch" revision="b0bd742551639c52030c070e5bf9390edbb536ba"/><project name="mccouch" path="mccouch" revision="88701cc326bc3dde4ed072bb8441be83adcfb2a5"/><project name="testrunner" path="testrunner" revision="f7a0a23b32c73866444b3a795fad647c350f47d9"/><project name="otp" path="otp" revision="b6dc1a844eab061d0a7153d46e7e68296f15a504" remote="erlang"/><project name="icu4c" path="icu4c" revision="26359393672c378f41f2103a8699c4357c894be7" remote="couchbase"/><project name="snappy" path="snappy" revision="5681dde156e9d07adbeeab79666c9a9d7a10ec95" remote="couchbase"/><project name="v8" path="v8" revision="447decb75060a106131ab4de934bcc374648e7f2" remote="couchbase"/><project name="gperftools" path="gperftools" revision="8f60ba949fb8576c530ef4be148bff97106ddc59" remote="couchbase"/><project name="pysqlite" path="pysqlite" revision="0ff6e32ea05037fddef1eb41a648f2a2141009ea" remote="couchbase"/></manifest>
Description
Staeps to repro: incrementally rebalance in 6 nodes
2012-10-17 17:48:05.545 ns_port_server:0:info:message(ns_1@10.1.3.84) - Port server memcached on node 'ns_1@10.1.3.84' exited with status 255. Restarting. Messages: Wed Oct 17 17:47:54.732679 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_10 - disconnected
Wed Oct 17 17:47:54.920179 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_11 - disconnected
Wed Oct 17 17:47:55.045179 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_12 - disconnected
Wed Oct 17 17:47:55.217054 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_13 - disconnected
Wed Oct 17 17:47:55.373304 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_14 - disconnected
Wed Oct 17 17:47:55.545179 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_15 - disconnected
Wed Oct 17 17:47:55.623304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_8"
Wed Oct 17 17:47:55.623304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_9"
Wed Oct 17 17:47:55.623304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_10"
Wed Oct 17 17:47:55.623304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_11"
Wed Oct 17 17:47:55.623304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_12"
Wed Oct 17 17:47:55.623304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_13"
Wed Oct 17 17:47:55.623304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_14"
Wed Oct 17 17:47:55.623304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_15"
Wed Oct 17 17:47:55.717054 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_16 - disconnected
Wed Oct 17 17:47:55.842054 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_17 - disconnected
Wed Oct 17 17:47:55.967054 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_18 - disconnected
Wed Oct 17 17:47:56.123304 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_19 - disconnected
Wed Oct 17 17:47:56.310804 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_20 - disconnected
Wed Oct 17 17:47:56.357679 Pacific Daylight Time 3: persistence cursor offset is negative. Reset it to 0.
Wed Oct 17 17:47:56.435804 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_21 - disconnected
Wed Oct 17 17:47:56.670179 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_22 - disconnected
Wed Oct 17 17:47:56.873304 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_23 - disconnected
Wed Oct 17 17:47:57.076429 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_24 - disconnected
Wed Oct 17 17:47:57.248304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_16"
Wed Oct 17 17:47:57.248304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_17"
Wed Oct 17 17:47:57.248304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_18"
Wed Oct 17 17:47:57.248304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_19"
Wed Oct 17 17:47:57.248304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_20"
Wed Oct 17 17:47:57.248304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_21"
Wed Oct 17 17:47:57.248304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_22"
Wed Oct 17 17:47:57.248304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_23"
Wed Oct 17 17:47:57.248304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_24"
Wed Oct 17 17:47:57.248304 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_25 - disconnected
Wed Oct 17 17:47:57.435804 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_26 - disconnected
Wed Oct 17 17:47:57.623304 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_27 - disconnected
Wed Oct 17 17:47:57.857679 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_28 - disconnected
Wed Oct 17 17:47:58.076429 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_29 - disconnected
Wed Oct 17 17:47:58.263929 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_30 - disconnected
Wed Oct 17 17:47:58.498304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_25"
Wed Oct 17 17:47:58.498304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_26"
Wed Oct 17 17:47:58.498304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_27"
Wed Oct 17 17:47:58.498304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_28"
Wed Oct 17 17:47:58.498304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_29"
Wed Oct 17 17:47:58.498304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_30"
Wed Oct 17 17:47:58.529554 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_31 - disconnected
Wed Oct 17 17:47:58.779554 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_32 - disconnected
Wed Oct 17 17:47:58.857679 Pacific Daylight Time 3: persistence cursor offset is negative. Reset it to 0.
Wed Oct 17 17:47:58.982679 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_33 - disconnected
Wed Oct 17 17:47:59.170179 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_34 - disconnected
Wed Oct 17 17:47:59.295179 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_35 - disconnected
Wed Oct 17 17:47:59.451429 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_36 - disconnected
Wed Oct 17 17:47:59.638929 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_37 - disconnected
Wed Oct 17 17:47:59.888929 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_38 - disconnected
Wed Oct 17 17:47:59.998304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_31"
Wed Oct 17 17:47:59.998304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_32"
Wed Oct 17 17:47:59.998304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_33"
Wed Oct 17 17:47:59.998304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_34"
Wed Oct 17 17:47:59.998304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_35"
Wed Oct 17 17:47:59.998304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_36"
Wed Oct 17 17:47:59.998304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_37"
Wed Oct 17 17:47:59.998304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_38"
Wed Oct 17 17:48:00.154554 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_39 - disconnected
Wed Oct 17 17:48:00.435804 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_40 - disconnected
Wed Oct 17 17:48:00.638929 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_41 - disconnected
Wed Oct 17 17:48:00.810804 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_42 - disconnected
Wed Oct 17 17:48:01.029554 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_43 - disconnected
Wed Oct 17 17:48:01.201429 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_44 - disconnected
Wed Oct 17 17:48:01.388929 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_45 - disconnected
Wed Oct 17 17:48:01.560804 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_46 - disconnected
Wed Oct 17 17:48:01.685804 Pacific Daylight Time 3: persistence cursor offset is negative. Reset it to 0.
Wed Oct 17 17:48:01.763929 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_47 - disconnected
Wed Oct 17 17:48:01.951429 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_48 - disconnected
Wed Oct 17 17:48:01.998304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_39"
Wed Oct 17 17:48:01.998304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_40"
Wed Oct 17 17:48:01.998304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_41"
Wed Oct 17 17:48:01.998304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_42"
Wed Oct 17 17:48:01.998304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_43"
Wed Oct 17 17:48:01.998304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_44"
Wed Oct 17 17:48:01.998304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_45"
Wed Oct 17 17:48:01.998304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_46"
Wed Oct 17 17:48:01.998304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_47"
Wed Oct 17 17:48:01.998304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_48"
Wed Oct 17 17:48:01.998304 Pacific Daylight Time 3: persistence cursor offset is negative. Reset it to 0.
Wed Oct 17 17:48:02.138929 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_49 - disconnected
Wed Oct 17 17:48:02.482679 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_50 - disconnected
Wed Oct 17 17:48:02.717054 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_51 - disconnected
Wed Oct 17 17:48:02.951429 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_52 - disconnected
Wed Oct 17 17:48:03.060804 Pacific Daylight Time 3: persistence cursor offset is negative. Reset it to 0.
Wed Oct 17 17:48:03.170179 Pacific Daylight Time 3: TAP (Consumer) eq_tapq:anon_53 - disconnected
Wed Oct 17 17:48:03.279554 Pacific Daylight Time 3: persistence cursor offset is negative. Reset it to 0.
./src/thread_cache.h:355] ptr != list->Next()
This application has requested the Runtime to terminate it in an unusual way.
Please contact the application's support team for more information.
Wed Oct 17 17:48:03.998304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_49"
Wed Oct 17 17:48:03.998304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_50"
Wed Oct 17 17:48:03.998304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_51"
Wed Oct 17 17:48:03.998304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_52"
Wed Oct 17 17:48:03.998304 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_53"
2012-10-17 17:48:05.561 ns_memcached:4:info:message(ns_1@10.1.3.84) - Control connection to memcached on 'ns_1@10.1.3.84' disconnected: {badmatch,
{error,
closed}}
2012-10-17 17:48:06.380 ns_port_server:0:info:message(ns_1@10.1.3.82) - Port server memcached on node 'ns_1@10.1.3.82' exited with status 255. Restarting. Messages: Wed Oct 17 17:48:01.498640 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_44_'ns_1@10.1.3.84' - Schedule the backfill for vbucket 44
Wed Oct 17 17:48:01.498640 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_44_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0
Wed Oct 17 17:48:01.498640 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_44_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0
Wed Oct 17 17:48:01.498640 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_44_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 44
Wed Oct 17 17:48:01.514265 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_44_'ns_1@10.1.3.84' - Backfill is completed with VBuckets 44,
Wed Oct 17 17:48:01.514265 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_44_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 44
Wed Oct 17 17:48:01.561140 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_44_'ns_1@10.1.3.84' - disconnected, keep alive for 300 seconds
Wed Oct 17 17:48:01.561140 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_44_'ns_1@10.1.3.84' - Connection is closed by force.
Wed Oct 17 17:48:01.592390 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_1467"
Wed Oct 17 17:48:01.592390 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_ns_1@10.1.3.84 - Sending TAP_OPAQUE with command "complete_vb_filter_change" and vbucket 0
Wed Oct 17 17:48:01.592390 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_44_'ns_1@10.1.3.84' - Clear the tap queues by force
Wed Oct 17 17:48:01.670515 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_45_'ns_1@10.1.3.84' - Schedule the backfill for vbucket 45
Wed Oct 17 17:48:01.670515 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_45_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0
Wed Oct 17 17:48:01.670515 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_45_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0
Wed Oct 17 17:48:01.670515 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_45_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 45
Wed Oct 17 17:48:01.686140 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_45_'ns_1@10.1.3.84' - Backfill is completed with VBuckets 45,
Wed Oct 17 17:48:01.686140 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_45_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 45
Wed Oct 17 17:48:01.764265 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_45_'ns_1@10.1.3.84' - disconnected, keep alive for 300 seconds
Wed Oct 17 17:48:01.764265 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_45_'ns_1@10.1.3.84' - Connection is closed by force.
Wed Oct 17 17:48:01.795515 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_1468"
Wed Oct 17 17:48:01.795515 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_ns_1@10.1.3.84 - Sending TAP_OPAQUE with command "complete_vb_filter_change" and vbucket 0
Wed Oct 17 17:48:01.795515 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_45_'ns_1@10.1.3.84' - Clear the tap queues by force
Wed Oct 17 17:48:01.858015 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_46_'ns_1@10.1.3.84' - Schedule the backfill for vbucket 46
Wed Oct 17 17:48:01.858015 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_46_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0
Wed Oct 17 17:48:01.889265 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_46_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0
Wed Oct 17 17:48:01.889265 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_46_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 46
Wed Oct 17 17:48:01.904890 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_46_'ns_1@10.1.3.84' - Backfill is completed with VBuckets 46,
Wed Oct 17 17:48:01.904890 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_46_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 46
Wed Oct 17 17:48:01.951765 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_46_'ns_1@10.1.3.84' - disconnected, keep alive for 300 seconds
Wed Oct 17 17:48:01.951765 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_46_'ns_1@10.1.3.84' - Connection is closed by force.
Wed Oct 17 17:48:01.983015 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_1469"
Wed Oct 17 17:48:01.983015 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_ns_1@10.1.3.84 - Sending TAP_OPAQUE with command "complete_vb_filter_change" and vbucket 0
Wed Oct 17 17:48:01.983015 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_46_'ns_1@10.1.3.84' - Clear the tap queues by force
Wed Oct 17 17:48:02.061140 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_47_'ns_1@10.1.3.84' - Schedule the backfill for vbucket 47
Wed Oct 17 17:48:02.061140 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_47_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0
Wed Oct 17 17:48:02.061140 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_47_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0
Wed Oct 17 17:48:02.061140 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_47_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 47
Wed Oct 17 17:48:02.076765 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_47_'ns_1@10.1.3.84' - Backfill is completed with VBuckets 47,
Wed Oct 17 17:48:02.076765 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_47_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 47
Wed Oct 17 17:48:02.139265 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_47_'ns_1@10.1.3.84' - disconnected, keep alive for 300 seconds
Wed Oct 17 17:48:02.186140 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_47_'ns_1@10.1.3.84' - Connection is closed by force.
Wed Oct 17 17:48:02.217390 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_1470"
Wed Oct 17 17:48:02.217390 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_ns_1@10.1.3.84 - Sending TAP_OPAQUE with command "complete_vb_filter_change" and vbucket 0
Wed Oct 17 17:48:02.217390 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_47_'ns_1@10.1.3.84' - Clear the tap queues by force
Wed Oct 17 17:48:02.326765 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_48_'ns_1@10.1.3.84' - Schedule the backfill for vbucket 48
Wed Oct 17 17:48:02.326765 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_48_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0
Wed Oct 17 17:48:02.326765 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_48_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0
Wed Oct 17 17:48:02.326765 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_48_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 48
Wed Oct 17 17:48:02.342390 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_48_'ns_1@10.1.3.84' - Backfill is completed with VBuckets 48,
Wed Oct 17 17:48:02.342390 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_48_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 48
Wed Oct 17 17:48:02.483015 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_48_'ns_1@10.1.3.84' - disconnected, keep alive for 300 seconds
Wed Oct 17 17:48:02.483015 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_48_'ns_1@10.1.3.84' - Connection is closed by force.
Wed Oct 17 17:48:02.514265 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_1471"
Wed Oct 17 17:48:02.514265 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_ns_1@10.1.3.84 - Sending TAP_OPAQUE with command "complete_vb_filter_change" and vbucket 0
Wed Oct 17 17:48:02.514265 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_48_'ns_1@10.1.3.84' - Clear the tap queues by force
Wed Oct 17 17:48:02.608015 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_49_'ns_1@10.1.3.84' - Schedule the backfill for vbucket 49
Wed Oct 17 17:48:02.608015 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_49_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0
Wed Oct 17 17:48:02.608015 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_49_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0
Wed Oct 17 17:48:02.639265 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_49_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 49
Wed Oct 17 17:48:02.654890 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_49_'ns_1@10.1.3.84' - Backfill is completed with VBuckets 49,
Wed Oct 17 17:48:02.654890 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_49_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 49
Wed Oct 17 17:48:02.717390 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_49_'ns_1@10.1.3.84' - disconnected, keep alive for 300 seconds
Wed Oct 17 17:48:02.717390 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_49_'ns_1@10.1.3.84' - Connection is closed by force.
Wed Oct 17 17:48:02.764265 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_1472"
Wed Oct 17 17:48:02.764265 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_ns_1@10.1.3.84 - Sending TAP_OPAQUE with command "complete_vb_filter_change" and vbucket 0
Wed Oct 17 17:48:02.764265 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_49_'ns_1@10.1.3.84' - Clear the tap queues by force
Wed Oct 17 17:48:02.842390 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_50_'ns_1@10.1.3.84' - Schedule the backfill for vbucket 50
Wed Oct 17 17:48:02.842390 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_50_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0
Wed Oct 17 17:48:02.858015 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_50_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0
Wed Oct 17 17:48:02.858015 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_50_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 50
Wed Oct 17 17:48:02.873640 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_50_'ns_1@10.1.3.84' - Backfill is completed with VBuckets 50,
Wed Oct 17 17:48:02.873640 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_50_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 50
Wed Oct 17 17:48:02.951765 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_50_'ns_1@10.1.3.84' - disconnected, keep alive for 300 seconds
Wed Oct 17 17:48:02.983015 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_50_'ns_1@10.1.3.84' - Connection is closed by force.
Wed Oct 17 17:48:03.014265 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_1473"
Wed Oct 17 17:48:03.014265 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_ns_1@10.1.3.84 - Sending TAP_OPAQUE with command "complete_vb_filter_change" and vbucket 0
Wed Oct 17 17:48:03.014265 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_50_'ns_1@10.1.3.84' - Clear the tap queues by force
Wed Oct 17 17:48:03.108015 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_51_'ns_1@10.1.3.84' - Schedule the backfill for vbucket 51
Wed Oct 17 17:48:03.108015 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_51_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0
Wed Oct 17 17:48:03.108015 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_51_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0
Wed Oct 17 17:48:03.108015 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_51_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 51
Wed Oct 17 17:48:03.123640 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_51_'ns_1@10.1.3.84' - Backfill is completed with VBuckets 51,
Wed Oct 17 17:48:03.123640 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_51_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 51
Wed Oct 17 17:48:03.170515 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_51_'ns_1@10.1.3.84' - disconnected, keep alive for 300 seconds
Wed Oct 17 17:48:03.201765 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_51_'ns_1@10.1.3.84' - Connection is closed by force.
Wed Oct 17 17:48:03.233015 Pacific Daylight Time 3: Schedule cleanup of "eq_tapq:anon_1474"
Wed Oct 17 17:48:03.233015 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_ns_1@10.1.3.84 - Sending TAP_OPAQUE with command "complete_vb_filter_change" and vbucket 0
Wed Oct 17 17:48:03.233015 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_51_'ns_1@10.1.3.84' - Clear the tap queues by force
Wed Oct 17 17:48:03.342390 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_52_'ns_1@10.1.3.84' - Schedule the backfill for vbucket 52
Wed Oct 17 17:48:03.342390 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_52_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "opaque_enable_auto_nack" and vbucket 0
Wed Oct 17 17:48:03.358015 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_52_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "enable_checkpoint_sync" and vbucket 0
Wed Oct 17 17:48:03.358015 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_52_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "initial_vbucket_stream" and vbucket 52
Wed Oct 17 17:48:03.373640 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_52_'ns_1@10.1.3.84' - Backfill is completed with VBuckets 52,
Wed Oct 17 17:48:03.373640 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_52_'ns_1@10.1.3.84' - Sending TAP_OPAQUE with command "close_backfill" and vbucket 52
./src/thread_cache.h:355] ptr != list->Next()
This application has requested the Runtime to terminate it in an unusual way.
Please contact the application's support team for more information.
Wed Oct 17 17:48:05.842390 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_ns_1@10.1.3.84 - disconnected, keep alive for 300 seconds
Wed Oct 17 17:48:05.842390 Pacific Daylight Time 3: TAP (Producer) eq_tapq:replication_building_52_'ns_1@10.1.3.84' - disconnected, keep alive for 300 seconds
2012-10-17 17:48:07.067 ns_memcached:4:info:message(ns_1@10.1.3.82) - Control connection to memcached on 'ns_1@10.1.3.82' disconnected: {badmatch,
{error,
closed}}
2012-10-17 17:48:10.067 ns_memcached:1:info:message(ns_1@10.1.3.82) - Bucket "bucket-0" loaded on node 'ns_1@10.1.3.82' in 1 seconds.
2012-10-17 17:48:10.770 ns_vbucket_mover:0:critical:message(ns_1@10.1.3.82) - <0.23228.5> exited with {unexpected_exit,
{'EXIT',<0.23235.5>,
{{wait_checkpoint_persisted_failed,"bucket-0",52,2,
[{'ns_1@10.1.3.84',
{'EXIT',
{badmatch,{error,closed,
{gen_server,call,
['ns_memcached-bucket-0',
infinity]}},
{gen_server,call,
[{'janitor_agent-bucket-0','ns_1@10.1.3.84'},
{if_rebalance,<0.22076.5>,
{wait_checkpoint_persisted,52,2}},
infinity]}}}}]},
[{ns_single_vbucket_mover, '-wait_checkpoint_persisted_many/5-fun-1-',5}]}}}
2012-10-17 17:48:10.770 ns_orchestrator:2:info:message(ns_1@10.1.3.82) - Rebalance exited with reason {unexpected_exit,
{'EXIT',<0.23235.5>,
{{wait_checkpoint_persisted_failed,"bucket-0",
52,2,
[{'ns_1@10.1.3.84',
{'EXIT',
{badmatch,{error,closed,
{gen_server,call,
['ns_memcached-bucket-0',
{wait_for_checkpoint_persistence,52,2}
,
infinity]}},
{gen_server,call,
[
,
{if_rebalance,<0.22076.5>,
{wait_checkpoint_persisted,52,2}},
infinity]}}}}]},
[
]}}}