Details
-
Bug
-
Resolution: Fixed
-
Critical
-
4.5.0
-
Centos
-
Untriaged
-
-
Unknown
Description
1. Load 25M documents into a cluster. (Pillowfight was used for the test). Run background mutations throughout.
2. Failover a node.
3. Delta recover the node after few minutes.
4. Rebalance the system.
The rebalance failed. Following logs were seen in the logs UI
Failed to add node 10.6.2.168:8091 to cluster. Prepare join failed. Authentication failed. Verify username and password. ns_cluster 005 ns_1@10.6.2.164 11:59:30 PM Tue May 10, 2016
|
Node 'ns_1@10.6.2.164' saw that node 'ns_1@10.6.2.168' went down. Details: [{nodedown_reason,
|
connection_closed}] ns_node_disco 005 ns_1@10.6.2.164 11:58:16 PM Tue May 10, 2016
|
Node 'ns_1@10.6.2.167' saw that node 'ns_1@10.6.2.168' went down. Details: [{nodedown_reason,
|
connection_closed}] ns_node_disco 005 ns_1@10.6.2.167 11:58:16 PM Tue May 10, 2016
|
Node ejected: 'ns_1@10.6.2.168' from node: 'ns_1@10.6.2.164' menelaus_web 101 ns_1@10.6.2.164 11:58:15 PM Tue May 10, 2016
|
Node 'ns_1@10.6.2.168' is leaving cluster. ns_cluster 001 ns_1@10.6.2.168 11:58:15 PM Tue May 10, 2016
|
Node ns_1@10.6.2.168 joined cluster ns_cluster 003 ns_1@10.6.2.168 11:58:12 PM Tue May 10, 2016
|
Couchbase Server has started on web port 8091 on node 'ns_1@10.6.2.168'. Version: "4.5.0-2488-enterprise". menelaus_sup 001 ns_1@10.6.2.168 11:58:12 PM Tue May 10, 2016
|
Node 'ns_1@10.6.2.167' saw that node 'ns_1@10.6.2.168' came up. Tags: [] ns_node_disco 004 ns_1@10.6.2.167 11:58:08 PM Tue May 10, 2016
|
Node 'ns_1@10.6.2.164' saw that node 'ns_1@10.6.2.168' came up. Tags: [] ns_node_disco 004 ns_1@10.6.2.164 11:58:08 PM Tue May 10, 2016
|
Started node add transaction by adding node 'ns_1@10.6.2.168' to nodes_wanted (group: 0) ns_cluster 000 ns_1@10.6.2.164 11:58:07 PM Tue May 10, 2016
|
Node 'ns_1@10.6.2.167' saw that node 'ns_1@10.6.2.168' went down. Details: [{nodedown_reason,
|
connection_closed}] ns_node_disco 005 ns_1@10.6.2.167 2:28:50 PM Mon May 9, 2016
|
Node 'ns_1@10.6.2.164' saw that node 'ns_1@10.6.2.168' went down. Details: [{nodedown_reason,
|
connection_closed}] ns_node_disco 005 ns_1@10.6.2.164 2:28:50 PM Mon May 9, 2016
|
Shutting down bucket "default" on 'ns_1@10.6.2.168' for deletion ns_memcached 000 ns_1@10.6.2.168 2:28:50 PM Mon May 9, 2016
|
Reset auto-failover count auto_failover 000 ns_1@10.6.2.164 2:28:49 PM Mon May 9, 2016
|
Rebalance completed successfully.
|
ns_orchestrator 001 ns_1@10.6.2.164 2:28:49 PM Mon May 9, 2016
|
Node 'ns_1@10.6.2.168' is leaving cluster. ns_cluster 001 ns_1@10.6.2.168 2:28:49 PM Mon May 9, 2016
|
Bucket "default" rebalance does not seem to be swap rebalance ns_vbucket_mover 000 ns_1@10.6.2.164 12:46:04 PM Mon May 9, 2016
|
Started rebalancing bucket default ns_rebalancer 000 ns_1@10.6.2.164 12:46:04 PM Mon May 9, 2016
|
Starting rebalance, KeepNodes = ['ns_1@10.6.2.164','ns_1@10.6.2.167'], EjectNodes = ['ns_1@10.6.2.168'], Failed over and being ejected nodes = []; no delta recovery nodes
|
ns_orchestrator 004 ns_1@10.6.2.164 12:46:04 PM Mon May 9, 2016
|
Rebalance exited with reason {badarg,
|
[{lists,keyfind,
|
[{'ns_1@10.6.2.168',341},
|
1,
|
[{{'ns_1@10.6.2.164',341},{24408,76,dcp}},
|
{{'ns_1@10.6.2.164',854},{24438,20,dcp}},
|
{{'ns_1@10.6.2.164',855},{24411,70,dcp}},
|
{{'ns_1@10.6.2.164',856},{24438,59,dcp}},
|
{{'ns_1@10.6.2.164',857},{24435,73,dcp}},
|
{{'ns_1@10.6.2.164',858},{24409,69,dcp}},
|
{{'ns_1@10.6.2.164',859},{24404,58,dcp}},
|
{{'ns_1@10.6.2.164',860},{24432,79,dcp}},
|
{{'ns_1@10.6.2.164',861},{24412,31,dcp}},
|
{{'ns_1@10.6.2.164',862},{24390,73,dcp}},
|
{{'ns_1@10.6.2.164',863},{24401,59,dcp}},
|
{{'ns_1@10.6.2.164',864},{24389,76,dcp}},
|
{{'ns_1@10.6.2.164',865},{24401,76,dcp}},
|
{{'ns_1@10.6.2.164',866},{24432,61,dcp}},
|
{{'ns_1@10.6.2.164',867},{24412,6,dcp}},
|
{{'ns_1@10.6.2.164',868},{24409,21,dcp}},
|
{{'ns_1@10.6.2.164',869},{24405,18,dcp}},
|
{{'ns_1@10.6.2.164',870},{24437,81,dcp}},
|
{{'ns_1@10.6.2.164',871},{24434,58,dcp}},
|
{{'ns_1@10.6.2.164',872},{24438,48,dcp}},
|
{{'ns_1@10.6.2.164',873},{24410,2,dcp}},
|
{{'ns_1@10.6.2.164',874},{24391,18,dcp}},
|
{{'ns_1@10.6.2.164',875},{24407,19,dcp}},
|
{{'ns_1@10.6.2.164',876},{24433,46,dcp}},
|
{{'ns_1@10.6.2.164',877},{24433,5,dcp}},
|
{{'ns_1@10.6.2.164',878},{24401,20,dcp}},
|
{{'ns_1@10.6.2.164',879},{24402,74,dcp}},
|
{{'ns_1@10.6.2.164',880},{24401,5,dcp}},
|
{{'ns_1@10.6.2.164',881},{24401,74,dcp}},
|
{{'ns_1@10.6.2.164',882},{24433,97,dcp}},
|
{{'ns_1@10.6.2.164',883},{24435,83,dcp}},
|
{{'ns_1@10.6.2.164',884},{24391,65,dcp}},
|
{{'ns_1@10.6.2.164',885},{24408,35,dcp}},
|
{{'ns_1@10.6.2.164',886},{24438,80,dcp}},
|
{{'ns_1@10.6.2.164',887},{24409,62,dcp}},
|
{{'ns_1@10.6.2.164',888},{24437,45,dcp}},
|
{{'ns_1@10.6.2.164',889},{24433,4,dcp}},
|
{{'ns_1@10.6.2.164',890},{24409,26,dcp}},
|
{{'ns_1@10.6.2.164',891},{24406,87,dcp}},
|
{{'ns_1@10.6.2.164',892},{24432,82,dcp}},
|
{{'ns_1@10.6.2.164',893},{24413,19,dcp}},
|
{{'ns_1@10.6.2.164',894},{24389,8,dcp}},
|
{{'ns_1@10.6.2.164',895},{24399,13,dcp}},
|
{{'ns_1@10.6.2.164',896},{24434,9,dcp}},
|
{{'ns_1@10.6.2.164',897},{24437,69,dcp}},
|
{{'ns_1@10.6.2.164',898},{24404,57,dcp}},
|
{{'ns_1@10.6.2.164',899},{24409,51,dcp}},
|
{{'ns_1@10.6.2.164',900},{24413,20,dcp}},
|
{{'ns_1@10.6.2.164',901},{24432,75,dcp}},
|
{{'ns_1@10.6.2.164',902},{24401,73,dcp}},
|
{{'ns_1@10.6.2.164',903},{24390,71,dcp}},
|
{{'ns_1@10.6.2.164',904},{24402,43,dcp}},
|
{{'ns_1@10.6.2.164',905},{24401,23,dcp}},
|
{{'ns_1@10.6.2.164',906},{24434,8,dcp}},
|
{{'ns_1@10.6.2.164',907},{24433,84,dcp}},
|
{{'ns_1@10.6.2.164',908},{24406,62,dcp}},
|
{{'ns_1@10.6.2.164',909},{24391,65,dcp}},
|
{{'ns_1@10.6.2.164',910},{24410,8,dcp}},
|
{{'ns_1@10.6.2.164',911},{24438,40,dcp}},
|
{{'ns_1@10.6.2.164',912},{24408,52,dcp}},
|
{{'ns_1@10.6.2.164',913},{24438,30,dcp}},
|
{{'ns_1@10.6.2.164',914},{24406,58,dcp}},
|
{{'ns_1@10.6.2.164',915},{24391,23,dcp}},
|
{{'ns_1@10.6.2.164',916},{24434,18,dcp}},
|
{{'ns_1@10.6.2.164',917},{24433,46,dcp}},
|
{{'ns_1@10.6.2.164',918},{24401,63,dcp}},
|
{{'ns_1@10.6.2.164',919},{24401,70,dcp}},
|
{{'ns_1@10.6.2.164',920},{24400,80,dcp}},
|
{{'ns_1@10.6.2.164',921},{24389,31,dcp}},
|
{{'ns_1@10.6.2.164',922},{24412,61,dcp}},
|
{{'ns_1@10.6.2.164',923},{24432,69,dcp}},
|
{{'ns_1@10.6.2.164',924},{24406,66,dcp}},
|
{{'ns_1@10.6.2.164',925},{24409,39,dcp}},
|
{{'ns_1@10.6.2.164',926},{24432,6,dcp}},
|
{{'ns_1@10.6.2.164',927},{24437,82,dcp}},
|
{{'ns_1@10.6.2.164',928},{24434,84,dcp}},
|
{{'ns_1@10.6.2.164',929},{24437,49,dcp}},
|
{{'ns_1@10.6.2.164',930},{24406,64,dcp}},
|
{{'ns_1@10.6.2.164',931},{24410,60,dcp}},
|
{{'ns_1@10.6.2.164',932},{24413,79,dcp}},
|
{{'ns_1@10.6.2.164',933},{24432,38,dcp}},
|
{{'ns_1@10.6.2.164',934},{24399,60,dcp}},
|
{{'ns_1@10.6.2.164',935},{24389,51,dcp}},
|
{{'ns_1@10.6.2.164',936},{24401,50,dcp}},
|
{{'ns_1@10.6.2.164',937},{24401,82,dcp}},
|
{{'ns_1@10.6.2.164',938},{24434,4,dcp}},
|
{{'ns_1@10.6.2.164',939},{24433,58,dcp}},
|
{{'ns_1@10.6.2.164',940},{24408,60,dcp}},
|
{{'ns_1@10.6.2.164',941},{24392,31,dcp}},
|
{{'ns_1@10.6.2.164',942},{24410,7,dcp}},
|
{{'ns_1@10.6.2.164',943},{24438,3,dcp}},
|
{{'ns_1@10.6.2.164',944},{24410,23,dcp}},
|
{{'ns_1@10.6.2.164',945},{24438,36,dcp}},
|
{{'ns_1@10.6.2.164',946},{24407,32,dcp}},
|
{{'ns_1@10.6.2.164',947},{24391,3,dcp}},
|
{{'ns_1@10.6.2.164',948},{24433,67,dcp}},
|
{{'ns_1@10.6.2.164',949},{24433,55,dcp}},
|
{{'ns_1@10.6.2.164',950},{24403,8,dcp}},
|
{{'ns_1@10.6.2.164',951},{24401,38,dcp}},
|
{{'ns_1@10.6.2.164',952},{24400,24,dcp}},
|
{{'ns_1@10.6.2.164',953},{24389,71,dcp}},
|
{{'ns_1@10.6.2.164',954},{24412,67,dcp}},
|
{{'ns_1@10.6.2.164',955},{24432,39,dcp}},
|
{{'ns_1@10.6.2.164',956},{24404,59,dcp}},
|
{{'ns_1@10.6.2.164',957},{24409,113,dcp}},
|
{{'ns_1@10.6.2.164',958},{24432,37,dcp}},
|
{{'ns_1@10.6.2.164',959},{24437,10,dcp}},
|
{{'ns_1@10.6.2.164',960},{24432,20,dcp}},
|
{{'ns_1@10.6.2.164',961},{24412,62,dcp}},
|
{{'ns_1@10.6.2.164',962},{24389,25,dcp}},
|
{{'ns_1@10.6.2.164',963},{24399,81,dcp}},
|
{{'ns_1@10.6.2.164',964},{24437,80,dcp}},
|
{{'ns_1@10.6.2.164',965},{24432,81,dcp}},
|
{{'ns_1@10.6.2.164',966},{24409,21,dcp}},
|
{{'ns_1@10.6.2.164',967},{24404,11,dcp}},
|
{{'ns_1@10.6.2.164',968},{24391,6,dcp}},
|
{{'ns_1@10.6.2.164',969},{24406,99,dcp}},
|
{{'ns_1@10.6.2.164',970},{24438,77,dcp}},
|
{{'ns_1@10.6.2.164',971},{24408,19,dcp}},
|
{{'ns_1@10.6.2.164',972},{24401,24,dcp}},
|
{{'ns_1@10.6.2.164',973},{24401,81,dcp}},
|
{{'ns_1@10.6.2.164',974},{24433,26,dcp}},
|
{{'ns_1@10.6.2.164',975},{24433,5,dcp}},
|
{{'ns_1@10.6.2.164',976},{24433,79,dcp}},
|
{{'ns_1@10.6.2.164',977},{24433,19,dcp}},
|
{{'ns_1@10.6.2.164',978},{24401,29,dcp}},
|
{{'ns_1@10.6.2.164',979},{24401,49,dcp}},
|
{{'ns_1@10.6.2.164',980},{24438,80,dcp}},
|
{{'ns_1@10.6.2.164',981},{24408,5,dcp}},
|
{{'ns_1@10.6.2.164',982},{24391,37,dcp}},
|
{{'ns_1@10.6.2.164',983},{24406,38,dcp}},
|
{{'ns_1@10.6.2.164',984},{24409,78,dcp}},
|
{{'ns_1@10.6.2.164',985},{24404,56,dcp}},
|
{{'ns_1@10.6.2.164',986},{24437,7,dcp}},
|
{{'ns_1@10.6.2.164',987},{24432,48,dcp}},
|
{{'ns_1@10.6.2.164',988},{24389,45,dcp}},
|
{{'ns_1@10.6.2.164',989},{24399,48,dcp}},
|
{{'ns_1@10.6.2.164',990},{24432,60,dcp}},
|
{{'ns_1@10.6.2.164',991},{24412,48,dcp}},
|
{{'ns_1@10.6.2.164',992},{24432,59,dcp}},
|
{{'ns_1@10.6.2.164',993},{24412,7,dcp}},
|
{{'ns_1@10.6.2.164',994},{24389,72,dcp}},
|
{{'ns_1@10.6.2.164',995},{24399,42,dcp}},
|
{{'ns_1@10.6.2.164',996},{24437,10,dcp}},
|
{{'ns_1@10.6.2.164',997},{24432,23,dcp}},
|
{{'ns_1@10.6.2.164',998},{24409,23,dcp}},
|
{{'ns_1@10.6.2.164',999},{24404,36,dcp}},
|
{{'ns_1@10.6.2.164',1000},{24391,87,dcp}},
|
{{'ns_1@10.6.2.164',1001},{24406,53,dcp}},
|
{{'ns_1@10.6.2.164',1002},{24438,25,dcp}},
|
{{'ns_1@10.6.2.164',1003},{24408,58,dcp}},
|
{{'ns_1@10.6.2.164',1004},{24401,103,dcp}},
|
{{'ns_1@10.6.2.164',1005},{24401,56,dcp}},
|
{{'ns_1@10.6.2.164',1006},{24433,61,dcp}},
|
{{'ns_1@10.6.2.164',1007},{24433,23,dcp}},
|
{{'ns_1@10.6.2.164',1008},{24433,84,dcp}},
|
{{'ns_1@10.6.2.164',1009},{24433,84,dcp}},
|
{{'ns_1@10.6.2.164',1010},{24401,66,dcp}},
|
{{'ns_1@10.6.2.164',1011},{24401,66,dcp}},
|
{{'ns_1@10.6.2.164',1012},{24438,49,dcp}},
|
{{'ns_1@10.6.2.164',1013},{24408,30,dcp}},
|
{{'ns_1@10.6.2.164',1014},{24391,20,dcp}},
|
{{'ns_1@10.6.2.164',1015},{24406,21,dcp}},
|
{{'ns_1@10.6.2.164',1016},{24409,76,dcp}},
|
{{'ns_1@10.6.2.164',1017},{24404,11,dcp}},
|
{{'ns_1@10.6.2.164',1018},{24437,33,dcp}},
|
{{'ns_1@10.6.2.164',1019},{24432,89,dcp}},
|
{{'ns_1@10.6.2.164',1020},{24389,7,dcp}},
|
{{'ns_1@10.6.2.164',1021},{24399,23,dcp}},
|
{{'ns_1@10.6.2.164',1022},{24432,25,dcp}},
|
{{'ns_1@10.6.2.164',1023},{24412,65,dcp}},
|
{{'ns_1@10.6.2.167',342},{24392,78,dcp}},
|
{{'ns_1@10.6.2.167',343},{24409,25,dcp}},
|
{{'ns_1@10.6.2.167',344},{24409,73,dcp}},
|
{{'ns_1@10.6.2.167',345},{24407,46,dcp}},
|
{{'ns_1@10.6.2.167',346},{24437,8,dcp}},
|
{{'ns_1@10.6.2.167',347},{24432,11,dcp}},
|
{{'ns_1@10.6.2.167',348},{24389,22,dcp}},
|
{{'ns_1@10.6.2.167',349},{24400,89,dcp}},
|
{{'ns_1@10.6.2.167',350},{24432,63,dcp}},
|
{{'ns_1@10.6.2.167',351},{24414,64,dcp}},
|
{{'ns_1@10.6.2.167',352},{24433,11,dcp}},
|
{{'ns_1@10.6.2.167',353},{24414,12,dcp}},
|
{{'ns_1@10.6.2.167',354},{24389,69,dcp}},
|
{{'ns_1@10.6.2.167',355},{24399,76,dcp}},
|
{{'ns_1@10.6.2.167',356},{24439,71,dcp}},
|
{{'ns_1@10.6.2.167',357},{24434,84,dcp}},
|
{{'ns_1@10.6.2.167',358},{24409,45,dcp}},
|
{{'ns_1@10.6.2.167',359},{24406,76,dcp}},
|
{{'ns_1@10.6.2.167',360},{24394,10,dcp}},
|
{{'ns_1@10.6.2.167',361},{24409,7,dcp}},
|
{{'ns_1@10.6.2.167',362},{24438,50,dcp}},
|
{{'ns_1@10.6.2.167',363},{24410,78,dcp}},
|
{{'ns_1@10.6.2.167',364},{24401,30,dcp}},
|
{{'ns_1@10.6.2.167',365},{24401,41,dcp}},
|
{{'ns_1@10.6.2.167',366},{24433,41,dcp}},
|
{{'ns_1@10.6.2.167',367},{24435,41,dcp}},
|
{{'ns_1@10.6.2.167',368},{24433,21,dcp}},
|
{{'ns_1@10.6.2.167',369},{24433,49,dcp}},
|
{{'ns_1@10.6.2.167',370},{24402,51,dcp}},
|
{{'ns_1@10.6.2.167',371},{24403,31,dcp}},
|
{{'ns_1@10.6.2.167',372},{24438,34,dcp}},
|
{{'ns_1@10.6.2.167',373},{24411,26,dcp}},
|
{{'ns_1@10.6.2.167',374},{24391,65,dcp}},
|
{{'ns_1@10.6.2.167',375},{24407,60,dcp}},
|
{{'ns_1@10.6.2.167',376},{24410,22,dcp}},
|
{{'ns_1@10.6.2.167',377},{24405,45,dcp}},
|
{{'ns_1@10.6.2.167',378},{24440,93,dcp}},
|
{{'ns_1@10.6.2.167',379},{24435,29,dcp}},
|
{{'ns_1@10.6.2.167',380},{24389,77,dcp}},
|
{{'ns_1@10.6.2.167',381},{24401,10,dcp}},
|
{{'ns_1@10.6.2.167',382},{24432,54,dcp}},
|
{{'ns_1@10.6.2.167',383},{24412,8,dcp}},
|
{{'ns_1@10.6.2.167',384},{24407,15,dcp}},
|
{{'ns_1@10.6.2.167',385},{24409,14,dcp}},
|
{{'ns_1@10.6.2.167',386},{24432,89,dcp}},
|
{{'ns_1@10.6.2.167',387},{24437,84,dcp}},
|
{{'ns_1@10.6.2.167',388},{24400,26,dcp}},
|
{{'ns_1@10.6.2.167',389},{24389,73,dcp}},
|
{{'ns_1@10.6.2.167',390},{24414,80,dcp}},
|
{{'ns_1@10.6.2.167',391},{24432,36,dcp}},
|
{{'ns_1@10.6.2.167',392},{24435,82,dcp}},
|
{{'ns_1@10.6.2.167',393},{24434,10,dcp}},
|
{{'ns_1@10.6.2.167',394},{24402,99,dcp}},
|
{{'ns_1@10.6.2.167',395},{24401,31,dcp}},
|
{{'ns_1@10.6.2.167',396},{24408,22,dcp}},
|
{{'ns_1@10.6.2.167',397},{24438,77,dcp}},
|
{{'ns_1@10.6.2.167',398},{24409,53,dcp}},
|
{{'ns_1@10.6.2.167',399},{24392,37,dcp}},
|
{{'ns_1@10.6.2.167',400},{24406,10,dcp}},
|
{{'ns_1@10.6.2.167',401},{24391,35,dcp}},
|
{{'ns_1@10.6.2.167',402},{24411,28,dcp}},
|
{{'ns_1@10.6.2.167',403},{24438,25,dcp}},
|
{{'ns_1@10.6.2.167',404},{24403,22,dcp}},
|
{{'ns_1@10.6.2.167',405},{24402,12,dcp}},
|
{{'ns_1@10.6.2.167',406},{24434,96,dcp}},
|
{{'ns_1@10.6.2.167',407},{24434,49,dcp}},
|
{{'ns_1@10.6.2.167',408},{24413,21,dcp}},
|
{{'ns_1@10.6.2.167',409},{24432,9,dcp}},
|
{{'ns_1@10.6.2.167',410},{24401,52,dcp}},
|
{{'ns_1@10.6.2.167',411},{24390,9,dcp}},
|
{{'ns_1@10.6.2.167',412},{24434,19,dcp}},
|
{{'ns_1@10.6.2.167',413},{24438,16,dcp}},
|
{{'ns_1@10.6.2.167',414},{24404,45,dcp}},
|
{{'ns_1@10.6.2.167',415},{24409,42,dcp}},
|
{{'ns_1@10.6.2.167',416},{24405,56,dcp}},
|
{{'ns_1@10.6.2.167',417},{24409,82,dcp}},
|
{{'ns_1@10.6.2.167',418},{24435,99,dcp}},
|
{{'ns_1@10.6.2.167',419},{24438,92,dcp}},
|
{{'ns_1@10.6.2.167',420},{24401,15,dcp}},
|
{{'ns_1@10.6.2.167',421},{24390,50,dcp}},
|
{{'ns_1@10.6.2.167',422},{24412,2,dcp}},
|
{{'ns_1@10.6.2.167',423},{24432,72,dcp}},
|
{{'ns_1@10.6.2.167',424},{24433,72,dcp}},
|
{{'ns_1@10.6.2.167',425},{24433,29,dcp}},
|
{{'ns_1@10.6.2.167',426},{24403,57,dcp}},
|
{{'ns_1@10.6.2.167',427},{24401,70,dcp}},
|
{{'ns_1@10.6.2.167',428},{24410,24,dcp}},
|
{{'ns_1@10.6.2.167',429},{24439,6,dcp}},
|
{{'ns_1@10.6.2.167',430},{24407,8,dcp}},
|
{{'ns_1@10.6.2.167',431},{24391,10,dcp}},
|
{{'ns_1@10.6.2.167',432},{24409,8,dcp}},
|
{{'ns_1@10.6.2.167',433},{24392,32,dcp}},
|
{{'ns_1@10.6.2.167',434},{24410,60,dcp}},
|
{{'ns_1@10.6.2.167',435},{24438,87,dcp}},
|
{{'ns_1@10.6.2.167',436},{24401,26,dcp}},
|
{{'ns_1@10.6.2.167',437},{24401,13,dcp}},
|
{{'ns_1@10.6.2.167',438},{24435,65,dcp}},
|
{{'ns_1@10.6.2.167',439},{24434,59,dcp}},
|
{{'ns_1@10.6.2.167',440},{24414,12,dcp}},
|
{{'ns_1@10.6.2.167',441},{24432,11,dcp}},
|
{{'ns_1@10.6.2.167',442},{24399,6,dcp}},
|
{{'ns_1@10.6.2.167',443},{24389,33,dcp}},
|
{{'ns_1@10.6.2.167',444},{24434,26,dcp}},
|
{{'ns_1@10.6.2.167',445},{24439,23,dcp}},
|
{{'ns_1@10.6.2.167',446},{24407,42,dcp}},
|
{{'ns_1@10.6.2.167',447},{24409,56,dcp}},
|
{{'ns_1@10.6.2.167',448},{24391,74,dcp}},
|
{{'ns_1@10.6.2.167',449},{24399,46,dcp}},
|
{{'ns_1@10.6.2.167',450},{24433,13,dcp}},
|
{{'ns_1@10.6.2.167',451},{24412,53,dcp}},
|
{{'ns_1@10.6.2.167',452},{24409,94,dcp}},
|
{{'ns_1@10.6.2.167',453},{24404,89,dcp}},
|
{{'ns_1@10.6.2.167',454},{24440,30,dcp}},
|
{{'ns_1@10.6.2.167',455},{24433,9,dcp}},
|
{{'ns_1@10.6.2.167',456},{24441,12,dcp}},
|
{{'ns_1@10.6.2.167',457},{24409,66,dcp}},
|
{{'ns_1@10.6.2.167',458},{24391,47,dcp}},
|
{{'ns_1@10.6.2.167',459},{24406,80,dcp}},
|
{{'ns_1@10.6.2.167',460},{24434,62,dcp}},
|
{{'ns_1@10.6.2.167',461},{24434,63,dcp}},
|
{{'ns_1@10.6.2.167',462},{24403,78,dcp}},
|
{{'ns_1@10.6.2.167',463},{24402,82,dcp}},
|
{{'ns_1@10.6.2.167',464},{24402,97,dcp}},
|
{{'ns_1@10.6.2.167',465},{24402,20,dcp}},
|
{{'ns_1@10.6.2.167',466},{24435,33,dcp}},
|
{{'ns_1@10.6.2.167',467},{24433,90,dcp}},
|
{{'ns_1@10.6.2.167',468},{24394,13,dcp}},
|
{{'ns_1@10.6.2.167',469},{24407,32,dcp}},
|
{{'ns_1@10.6.2.167',470},{24438,99,dcp}},
|
{{'ns_1@10.6.2.167',471},{24408,29,dcp}},
|
{{'ns_1@10.6.2.167',472},{24437,80,dcp}},
|
{{'ns_1@10.6.2.167',473},{24432,41,dcp}},
|
{{'ns_1@10.6.2.167',474},{24412,83,dcp}},
|
{{'ns_1@10.6.2.167',475},{24405,82,dcp}},
|
{{'ns_1@10.6.2.167',476},{24434,67,dcp}},
|
{{'ns_1@10.6.2.167',477},{24413,81,dcp}},
|
{{'ns_1@10.6.2.167',478},{24390,14,dcp}},
|
{{'ns_1@10.6.2.167',479},{24399,51,dcp}},
|
{{'ns_1@10.6.2.167',480},{24389,42,dcp}},
|
{{'ns_1@10.6.2.167',481},{24399,49,dcp}},
|
{{'ns_1@10.6.2.167',482},{24434,66,dcp}},
|
{{'ns_1@10.6.2.167',483},{24413,52,dcp}},
|
{{'ns_1@10.6.2.167',484},{24412,51,dcp}},
|
{{'ns_1@10.6.2.167',485},{24404,81,dcp}},
|
{{'ns_1@10.6.2.167',486},{24439,15,dcp}},
|
{{'ns_1@10.6.2.167',487},{24432,7,dcp}},
|
{{'ns_1@10.6.2.167',488},{24440,9,dcp}},
|
{{'ns_1@10.6.2.167',489},{24408,28,dcp}},
|
{{'ns_1@10.6.2.167',490},{24394,12,dcp}},
|
{{'ns_1@10.6.2.167',491},{24407,41,dcp}},
|
{{'ns_1@10.6.2.167',492},{24434,12,dcp}},
|
{{'ns_1@10.6.2.167',493},{24433,44,dcp}},
|
{{'ns_1@10.6.2.167',494},{24401,52,dcp}},
|
{{'ns_1@10.6.2.167',495},{24401,43,dcp}},
|
{{'ns_1@10.6.2.167',496},{24403,13,dcp}},
|
{{'ns_1@10.6.2.167',497},{24402,39,dcp}},
|
{{'ns_1@10.6.2.167',498},{24433,32,dcp}},
|
{{'ns_1@10.6.2.167',499},{24433,27,dcp}},
|
{{'ns_1@10.6.2.167',500},{24392,11,dcp}},
|
{{'ns_1@10.6.2.167',501},{24406,70,dcp}},
|
{{'ns_1@10.6.2.167',502},{24441,61,dcp}},
|
{{'ns_1@10.6.2.167',503},{24409,61,dcp}},
|
{{'ns_1@10.6.2.167',504},{24438,106,dcp}},
|
{{'ns_1@10.6.2.167',505},{24432,71,dcp}},
|
{{'ns_1@10.6.2.167',506},{24410,34,dcp}},
|
{{'ns_1@10.6.2.167',507},{24405,93,dcp}},
|
{{'ns_1@10.6.2.167',508},{24432,11,dcp}},
|
{{'ns_1@10.6.2.167',509},{24412,50,dcp}},
|
{{'ns_1@10.6.2.167',510},{24391,54,dcp}},
|
{{'ns_1@10.6.2.167',511},{24400,7,dcp}}|
|
{'EXIT',
|
{{janitor_agent_servant_died,
|
{{error,closed},
|
{gen_server,call,
|
['ns_memcached-default',
|
{get_mass_dcp_docs_estimate,
|
[341,342,343,344,345,346,347,348,349,
|
350,351,352,353,354,355,356,357,358,
|
359,360,361,362,363,364,365,366,367,
|
368,369,370,371,372,373,374,375,376,
|
377,378,379,380,381,382,383,384,385,
|
386,387,388,389,390,391,392,393,394,
|
395,396,397,398,399,400,401,402,403,
|
404,405,406,407,408,409,410,411,412,
|
413,414,415,416,417,418,419,420,421,
|
422,423,424,425,426,427,428,429,430,
|
431,432,433,434,435,436,437,438,439,
|
440,441,442,443,444,445,446,447,448,
|
449,450,451,452,453,454,455,456,457,
|
458,459,460,461,462,463,464,465,466,
|
467,468,469,470,471,472,473,474,475,
|
476,477,478,479,480,481,482,483,484,
|
485,486,487,488,489,490,491,492,493,
|
494,495,496,497,498,499,500,501,502,
|
503,504,505,506,507,508,509,510,511,
|
854,855,856,857,858,859,860,861,862,
|
863,864,865,866,867,868,869,870,871,
|
872,873,874,875,876,877,878,879,880,
|
881,882,883,884,885,886,887,888,889,
|
890,891,892,893,894,895,896,897,898,
|
899,900,901,902,903,904,905,906,907,
|
908,909,910,911,912,913,914,915,916,
|
917,918,919,920,921,922,923,924,925,
|
926,927,928,929,930,931,932,933,934,
|
935,936,937,938,939,940,941,942,943,
|
944,945,946,947,948,949,950,951,952,
|
953,954,955,956,957,958,959,960,961,
|
962,963,964,965,966,967,968,969,970,
|
971,972,973,974,975,976,977,978,979,
|
980,981,982,983,984,985,986,987,988,
|
989,990,991,992,993,994,995,996,997,
|
998,999,1000,1001,1002,1003,1004,
|
1005,1006,1007,1008,1009,1010,1011,
|
1012,1013,1014,1015,1016,1017,1018,
|
1019,1020,1021,1022,1023]},
|
360000]}}},
|
[{janitor_agent,get_servant_call_reply,1,
|
[{file,"src/janitor_agent.erl"},
|
{line,532}]},
|
{janitor_agent,
|
get_mass_dcp_docs_estimate,3,
|
[{file,"src/janitor_agent.erl"},
|
{line,570}]},
|
{ns_rebalance_observer,
|
'-initiate_bucket_rebalance/2-fun-0-',3,
|
[{file,"src/ns_rebalance_observer.erl"},
|
{line,197}]},
|
{misc,'-parallel_map/3-fun-0-',4,
|
[{file,"src/misc.erl"},{line,89}]}]}}]],
|
[]},
|
{ns_rebalance_observer,
|
'-initiate_bucket_rebalance/2-lc$^7/1-7-',6,
|
[{file,"src/ns_rebalance_observer.erl"},
|
{line,218}]},
|
{ns_rebalance_observer,
|
'-initiate_bucket_rebalance/2-lc$^6/1-6-',2,
|
[{file,"src/ns_rebalance_observer.erl"},
|
{line,217}]},
|
{ns_rebalance_observer,
|
initiate_bucket_rebalance,2,
|
[{file,"src/ns_rebalance_observer.erl"},
|
{line,214}]},
|
{ns_rebalance_observer,handle_set_ff_map,2,
|
[{file,"src/ns_rebalance_observer.erl"},
|
{line,257}]},
|
{ns_rebalance_observer,handle_cast,2,
|
[{file,"src/ns_rebalance_observer.erl"},
|
{line,105}]},
|
{gen_server,handle_msg,5,
|
[{file,"gen_server.erl"},{line,604}]},
|
{proc_lib,init_p_do_apply,3,
|
[{file,"proc_lib.erl"},{line,239}]}]}
|
