{"vbucket": 818, "type": "dcpAddStream", "ts": 1645623899.730842, "streamType": "add", "side": "consumer", "pid": "<0.17047.108>", "opaque": 818, "node": "ns_1@172.23.107.221", "connectionName": "replication:ns_1@172.23.107.76->ns_1@172.23.107.221:GleamBookUsers0", "bucket": "GleamBookUsers0", "time": "2022-02-23 05:44:59.730842"} {"vbucket": 818, "type": "dcpAddStreamResponse", "ts": 1645623899.859819, "success": true, "status": "success", "side": "consumer", "rawStatus": 0, "pid": "<0.17047.108>", "opaque": 818, "node": "ns_1@172.23.107.221", "connectionName": "replication:ns_1@172.23.107.76->ns_1@172.23.107.221:GleamBookUsers0", "bucket": "GleamBookUsers0", "time": "2022-02-23 05:44:59.859819"} ## Rebalance starts. Adding .126 ejecting .76 {"type": "rebalanceStart", "ts": 1645624202.617244, "pid": "<0.5885.109>", "keepNodes": ["172.23.107.232:11209", "172.23.107.221:11209", "172.23.107.126:11209", "172.23.107.237:11209", "172.23.107.239:11209"], "failedNodes": [], "ejectNodes": ["172.23.107.76:11209"], "deltaNodes": [], "time": "2022-02-23 05:50:02.617244"} {"type": "rebalanceStageStarted", "ts": 1645624202.629451, "stage": ["kv"], "nodes": ["ns_1@172.23.107.232", "ns_1@172.23.107.221", "ns_1@172.23.107.126", "ns_1@172.23.107.237", "ns_1@172.23.107.239", "ns_1@172.23.107.76"], "time": "2022-02-23 05:50:02.629451"} {"type": "rebalanceStageStarted", "ts": 1645624202.664754, "stage": ["kv", "kv_delta_recovery"], "nodes": [], "time": "2022-02-23 05:50:02.664754"} {"type": "rebalanceStageCompleted", "ts": 1645624202.69088, "stage": ["kv", "kv_delta_recovery"], "time": "2022-02-23 05:50:02.690880"} {"type": "bucketRebalanceStarted", "ts": 1645624217.12132, "pid": "<0.5970.109>", "node": "ns_1@172.23.107.221", "bucket": "GleamBookUsers0", "time": "2022-02-23 05:50:17.121320"} {"vbucket": 0, "type": "updateFastForwardMap", "ts": 1645624217.174907, "bucket": "GleamBookUsers0", "chainBefore": [], "chainAfter": ["172.23.107.221:11209", "172.23.107.232:11209"], "time": "2022-02-23 05:50:17.174907"} {"vbucket": 818, "type": "updateFastForwardMap", "ts": 1645624217.174907, "bucket": "GleamBookUsers0", "chainBefore": [], "chainAfter": ["172.23.107.126:11209", "172.23.107.221:11209"], "time": "2022-02-23 05:50:17.174907"} ## Start vbucket move .76 {"vbucket": 818, "type": "vbucketMoveStart", "ts": 1645626137.345803, "pid": "<0.8361.113>", "node": "ns_1@172.23.107.76", "bucket": "GleamBookUsers0", "chainBefore": ["172.23.107.76:11209", "172.23.107.221:11209"], "chainAfter": ["172.23.107.126:11209", "172.23.107.221:11209"], "time": "2022-02-23 06:22:17.345803"} {"vbucket": 818, "type": "vbucketStateChange", "ts": 1645626137.347527, "state": "replica", "meta": "undefined", "host": "172.23.107.221:11209", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:22:17.347527"} {"vbucket": 818, "type": "vbucketStateChange", "ts": 1645626137.347833, "state": "replica", "meta": "undefined", "host": "172.23.107.126:11209", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:22:17.347833"} ## replication stream from .76 -> .126 {"vbucket": 818, "type": "dcpAddStream", "ts": 1645626137.351165, "streamType": "add", "side": "consumer", "pid": "<34363.24768.60>", "opaque": 818, "node": "ns_1@172.23.107.126", "connectionName": "replication:ns_1@172.23.107.76->ns_1@172.23.107.126:GleamBookUsers0", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:22:17.351165"} ## replication stream from .76 -> .221 {"vbucket": 818, "type": "dcpAddStream", "ts": 1645626137.354134, "streamType": "add", "side": "consumer", "pid": "<0.8594.113>", "opaque": 818, "node": "ns_1@172.23.107.221", "connectionName": "replication:ns_1@172.23.107.76->ns_1@172.23.107.221:GleamBookUsers0", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:22:17.354134"} {"vbucket": 818, "type": "dcpAddStreamResponse", "ts": 1645626137.36563, "success": true, "status": "success", "side": "consumer", "rawStatus": 0, "pid": "<0.8594.113>", "opaque": 818, "node": "ns_1@172.23.107.221", "connectionName": "replication:ns_1@172.23.107.76->ns_1@172.23.107.221:GleamBookUsers0", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:22:17.365630"} {"vbucket": 818, "type": "dcpAddStreamResponse", "ts": 1645626137.622894, "success": true, "status": "success", "side": "consumer", "rawStatus": 0, "pid": "<34363.24768.60>", "opaque": 818, "node": "ns_1@172.23.107.126", "connectionName": "replication:ns_1@172.23.107.76->ns_1@172.23.107.126:GleamBookUsers0", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:22:17.622894"} {"vbucket": 818, "type": "indexingInitiated", "ts": 1645626137.624946, "node": "172.23.107.126:11209", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:22:17.624946"} {"vbucket": 818, "type": "backfillPhaseStarted", "ts": 1645626137.624957, "bucket": "GleamBookUsers0", "time": "2022-02-23 06:22:17.624957"} {"vbucket": 818, "type": "seqnoWaitingStarted", "ts": 1645626181.160187, "seqno": 1207137, "node": "172.23.107.126:11209", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:01.160187"} {"vbucket": 818, "type": "seqnoWaitingStarted", "ts": 1645626181.160187, "seqno": 1207137, "node": "172.23.107.221:11209", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:01.160187"} {"vbucket": 818, "type": "seqnoWaitingEnded", "ts": 1645626181.304446, "seqno": 1207137, "node": "172.23.107.126:11209", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:01.304446"} {"vbucket": 818, "type": "seqnoWaitingEnded", "ts": 1645626181.304446, "seqno": 1207137, "node": "172.23.107.221:11209", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:01.304446"} {"vbucket": 818, "type": "backfillPhaseEnded", "ts": 1645626181.304513, "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:01.304513"} {"vbucket": 818, "type": "vbucketStateChange", "ts": 1645626181.305653, "state": "active", "meta": {"topology": [["ns_1@172.23.107.76", "ns_1@172.23.107.221"]]}, "host": "172.23.107.76:11209", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:01.305653"} {"vbucket": 818, "type": "seqnoWaitingStarted", "ts": 1645626181.307899, "seqno": 1207137, "node": "172.23.107.126:11209", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:01.307899"} {"vbucket": 818, "type": "seqnoWaitingStarted", "ts": 1645626181.307899, "seqno": 1207137, "node": "172.23.107.221:11209", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:01.307899"} {"vbucket": 818, "type": "seqnoWaitingEnded", "ts": 1645626181.31446, "seqno": 1207137, "node": "172.23.107.126:11209", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:01.314460"} {"vbucket": 818, "type": "seqnoWaitingEnded", "ts": 1645626181.31446, "seqno": 1207137, "node": "172.23.107.221:11209", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:01.314460"} {"vbucket": 818, "type": "waitIndexUpdatedStarted", "ts": 1645626181.314474, "node": "172.23.107.126:11209", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:01.314474"} {"vbucket": 818, "type": "waitIndexUpdatedEnded", "ts": 1645626181.315796, "node": "172.23.107.126:11209", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:01.315796"} {"vbucket": 818, "type": "vbucketStateChange", "ts": 1645626181.316678, "state": "active", "meta": {"topology": [["ns_1@172.23.107.76", "ns_1@172.23.107.221"], ["ns_1@172.23.107.126", "ns_1@172.23.107.221"]]}, "host": "172.23.107.76:11209", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:01.316678"} {"vbucket": 818, "type": "seqnoWaitingStarted", "ts": 1645626181.318685, "seqno": 1207137, "node": "172.23.107.126:11209", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:01.318685"} {"vbucket": 818, "type": "seqnoWaitingStarted", "ts": 1645626181.318685, "seqno": 1207137, "node": "172.23.107.221:11209", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:01.318685"} {"vbucket": 818, "type": "seqnoWaitingEnded", "ts": 1645626181.599839, "seqno": 1207137, "node": "172.23.107.126:11209", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:01.599839"} {"vbucket": 818, "type": "seqnoWaitingEnded", "ts": 1645626181.599839, "seqno": 1207137, "node": "172.23.107.221:11209", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:01.599839"} ## takeover started .126 {"vbucket": 818, "type": "takeoverStarted", "ts": 1645626181.599856, "oldMaster": "172.23.107.76:11209", "node": "172.23.107.126:11209", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:01.599856"} ## replication stream closed .76 -> .126 {"vbucket": 818, "type": "dcpCloseStream", "ts": 1645626181.600965, "side": "consumer", "pid": "<34363.24768.60>", "opaque": 818, "node": "ns_1@172.23.107.126", "connectionName": "replication:ns_1@172.23.107.76->ns_1@172.23.107.126:GleamBookUsers0", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:01.600965"} {"vbucket": 818, "type": "dcpCloseStream", "ts": 1645626181.601086, "side": "producer", "pid": "<34363.24762.60>", "opaque": 818, "node": "ns_1@172.23.107.126", "connectionName": "replication:ns_1@172.23.107.76->ns_1@172.23.107.126:GleamBookUsers0", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:01.601086"} ## close replication streams from .76 -> .126 {"vbucket": 818, "type": "dcpCloseStreamResponse", "ts": 1645626181.618332, "success": true, "status": "success", "side": "producer", "rawStatus": 0, "pid": "<34363.24768.60>", "opaque": 818, "node": "ns_1@172.23.107.126", "connectionName": "replication:ns_1@172.23.107.76->ns_1@172.23.107.126:GleamBookUsers0", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:01.618332"} {"vbucket": 818, "type": "dcpCloseStreamResponse", "ts": 1645626181.718472, "success": true, "status": "success", "side": "consumer", "rawStatus": 0, "pid": "<34363.24768.60>", "opaque": 818, "node": "ns_1@172.23.107.126", "connectionName": "replication:ns_1@172.23.107.76->ns_1@172.23.107.126:GleamBookUsers0", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:01.718472"} ## add replication stream from .76 -> .126 {"vbucket": 818, "type": "dcpAddStream", "ts": 1645626181.719808, "streamType": "takeover", "side": "consumer", "pid": "<34363.24768.60>", "opaque": 818, "node": "ns_1@172.23.107.126", "connectionName": "replication:ns_1@172.23.107.76->ns_1@172.23.107.126:GleamBookUsers0", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:01.719808"} i ## Set vbucket state -> pending -> active on .126 {"vbucket": 818, "type": "dcpSetVbucketState", "ts": 1645626181.989522, "state": "pending", "pid": "<34363.24768.60>", "node": "ns_1@172.23.107.126", "connectionName": "replication:ns_1@172.23.107.76->ns_1@172.23.107.126:GleamBookUsers0", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:01.989522"} {"vbucket": 818, "type": "dcpSetVbucketState", "ts": 1645626182.141104, "state": "active", "pid": "<34363.24768.60>", "node": "ns_1@172.23.107.126", "connectionName": "replication:ns_1@172.23.107.76->ns_1@172.23.107.126:GleamBookUsers0", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:02.141104"} ## takeover ended .126 {"vbucket": 818, "type": "takeoverEnded", "ts": 1645626182.141282, "oldMaster": "172.23.107.76:11209", "node": "172.23.107.126:11209", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:02.141282"} ## vbucket state -> active on .126 {"vbucket": 818, "type": "vbucketStateChange", "ts": 1645626182.142166, "state": "active", "meta": "undefined", "host": "172.23.107.126:11209", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:02.142166"} ## vbucket state -> replica on .221 {"vbucket": 818, "type": "vbucketStateChange", "ts": 1645626182.144631, "state": "replica", "meta": "undefined", "host": "172.23.107.221:11209", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:02.144631"} ## stream closed .76 -> .221 {"vbucket": 818, "type": "dcpCloseStream", "ts": 1645626182.145553, "side": "consumer", "pid": "<0.8594.113>", "opaque": 818, "node": "ns_1@172.23.107.221", "connectionName": "replication:ns_1@172.23.107.76->ns_1@172.23.107.221:GleamBookUsers0", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:02.145553"} {"vbucket": 818, "type": "dcpCloseStream", "ts": 1645626182.145772, "side": "producer", "pid": "<0.8271.113>", "opaque": 818, "node": "ns_1@172.23.107.221", "connectionName": "replication:ns_1@172.23.107.76->ns_1@172.23.107.221:GleamBookUsers0", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:02.145772"} ## close stream response .76 -> .221 {"vbucket": 818, "type": "dcpCloseStreamResponse", "ts": 1645626182.146073, "success": true, "status": "success", "side": "consumer", "rawStatus": 0, "pid": "<0.8594.113>", "opaque": 818, "node": "ns_1@172.23.107.221", "connectionName": "replication:ns_1@172.23.107.76->ns_1@172.23.107.221:GleamBookUsers0", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:02.146073"} {"vbucket": 818, "type": "dcpCloseStreamResponse", "ts": 1645626182.146443, "success": true, "status": "success", "side": "producer", "rawStatus": 0, "pid": "<0.8594.113>", "opaque": 818, "node": "ns_1@172.23.107.221", "connectionName": "replication:ns_1@172.23.107.76->ns_1@172.23.107.221:GleamBookUsers0", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:02.146443"} ## update map ["172.23.107.126:11209", "172.23.107.221:11209"] {"vbucket": 818, "type": "updateMap", "ts": 1645626182.153772, "bucket": "GleamBookUsers0", "chainBefore": ["172.23.107.76:11209", "172.23.107.221:11209"], "chainAfter": ["172.23.107.126:11209", "172.23.107.221:11209"], "time": "2022-02-23 06:23:02.153772"} ## vbucket state -> active with "meta": {"topology": [["ns_1@172.23.107.126", "ns_1@172.23.107.221"]]} {"vbucket": 818, "type": "vbucketStateChange", "ts": 1645626182.180557, "state": "active", "meta": {"topology": [["ns_1@172.23.107.126", "ns_1@172.23.107.221"]]}, "host": "172.23.107.126:11209", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:02.180557"} ## vbucket state -> replica on .221 {"vbucket": 818, "type": "vbucketStateChange", "ts": 1645626182.184096, "state": "replica", "meta": "undefined", "host": "172.23.107.221:11209", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:02.184096"} ## replicator start .126 -> .221 {"type": "dcpReplicatorStart", "ts": 1645626182.189648, "producerNode": "ns_1@172.23.107.126", "producerConn": "<0.10844.113>", "pid": "<0.10810.113>", "consumerNode": "ns_1@172.23.107.221", "consumerConn": "<0.11314.113>", "connectionName": "replication:ns_1@172.23.107.126->ns_1@172.23.107.221:GleamBookUsers0", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:02.189648"} ## replicator terminate .126 -> .221 {"type": "dcpReplicatorTerminate", "ts": 1645626182.18974, "reason": "{done,{state,[{<0.11314.113>,#Port<0.52100>},{<0.10844.113>,#Port<0.52101>}],\n <0.11314.113>,\n \"replication:ns_1@172.23.107.126->ns_1@172.23.107.221:GleamBookUsers0\",\n 'ns_1@172.23.107.126',\"GleamBookUsers0\"}}", "producerNode": "ns_1@172.23.107.126", "producerConn": "<0.10844.113>", "pid": "<0.10810.113>", "consumerNode": "ns_1@172.23.107.221", "consumerConn": "<0.11314.113>", "connectionName": "replication:ns_1@172.23.107.126->ns_1@172.23.107.221:GleamBookUsers0", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:02.189740"} ## stream added .126 -> .221 {"vbucket": 818, "type": "dcpAddStream", "ts": 1645626182.189816, "streamType": "add", "side": "consumer", "pid": "<0.11314.113>", "opaque": 818, "node": "ns_1@172.23.107.221", "connectionName": "replication:ns_1@172.23.107.126->ns_1@172.23.107.221:GleamBookUsers0", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:02.189816"} {"vbucket": 818, "type": "dcpAddStreamResponse", "ts": 1645626182.199729, "success": true, "status": "success", "side": "consumer", "rawStatus": 0, "pid": "<0.11314.113>", "opaque": 818, "node": "ns_1@172.23.107.221", "connectionName": "replication:ns_1@172.23.107.126->ns_1@172.23.107.221:GleamBookUsers0", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:02.199729"} ## Done with vbucket move {"vbucket": 818, "type": "vbucketMoveDone", "ts": 1645626182.202637, "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:02.202637"} {"vbucket": 818, "type": "vbucketMoverTerminate", "ts": 1645626182.202645, "reason": "normal", "pid": "<0.8361.113>", "node": "ns_1@172.23.107.221", "bucket": "GleamBookUsers0", "time": "2022-02-23 06:23:02.202645"}