UPR Issue: Failover logs are inconsistent (active vs replica) after rebalance-in operation

3.0.0-641, centos-64 bit

Nodes: 10.6.2.144,10.6.2.145, 10.6.2.146, and 10.6.2.147

1. Add 3 node to cluster
2. Create default bucket, replica=1
3. Add 10K items
4. Wait for data to persist and replication to complete
5. Rebalance-in 1 node '10.6.2.147'
6. Wait for data to persist and replication to complete

Observation:: After Step 6 we found inconsistency in failover logs for active and replica

bucket default, vbucket vb_443 :: Original node 10.6.2.147 replica :: UUID 272555538338574, Change node 10.6.2.144 active UUID 198791938697315

bucket default, vbucket vb_440 :: Original node 10.6.2.147 replica :: seq 0, Change node 10.6.2.144 active :: seq 30

bucket default, vbucket vb_447 :: Original node 10.6.2.147 replica :: UUID 68701960302728, Change node 10.6.2.144 active UUID 187590445270116

bucket default, vbucket vb_446 :: Original node 10.6.2.147 replica :: UUID 85589133859170, Change node 10.6.2.144 active UUID 147724114922597

bucket default, vbucket vb_847 :: Original node 10.6.2.147 active :: seq 14, Change node 10.6.2.144 replica :: seq 0

bucket default, vbucket vb_846 :: Original node 10.6.2.147 active :: UUID 61842000313772, Change node 10.6.2.144 replica UUID 128495901897036

bucket default, vbucket vb_844 :: Original node 10.6.2.147 active :: seq 28, Change node 10.6.2.144 replica :: seq 0

bucket default, vbucket vb_843 :: Original node 10.6.2.147 active :: UUID 55256767097625, Change node 10.6.2.144 replica UUID 78541218331652

bucket default, vbucket vb_470 :: Original node 10.6.2.147 replica :: UUID 165447412416565, Change node 10.6.2.144 active UUID 1222713356917

bucket default, vbucket vb_808 :: Original node 10.6.2.147 active :: UUID 191307389958083, Change node 10.6.2.144 replica UUID 90659987816570

bucket default, vbucket vb_808 :: Original node 10.6.2.147 active :: seq 0, Change node 10.6.2.144 replica :: seq 32

bucket default, vbucket vb_771 :: Original node 10.6.2.147 active :: UUID 118279730140163, Change node 10.6.2.144 replica UUID 223031757349835

bucket default, vbucket vb_804 :: Original node 10.6.2.147 active :: UUID 122186041527705, Change node 10.6.2.144 replica UUID 100874246216672

bucket default, vbucket vb_772 :: Original node 10.6.2.147 active :: UUID 166357089928871, Change node 10.6.2.144 replica UUID 125868264023720

bucket default, vbucket vb_507 :: Original node 10.6.2.147 replica :: UUID 140466118804888, Change node 10.6.2.144 active UUID 274459990485683

bucket default, vbucket vb_505 :: Original node 10.6.2.147 replica :: UUID 205434444527042, Change node 10.6.2.144 active UUID 28484141437592

bucket default, vbucket vb_501 :: Original node 10.6.2.147 replica :: UUID 235515663568787, Change node 10.6.2.144 active UUID 220192958194238

bucket default, vbucket vb_501 :: Original node 10.6.2.147 replica :: seq 8, Change node 10.6.2.144 active :: seq 0

bucket default, vbucket vb_811 :: Original node 10.6.2.147 active :: seq 4, Change node 10.6.2.144 replica :: seq 0

bucket default, vbucket vb_812 :: Original node 10.6.2.147 active :: UUID 85681576080629, Change node 10.6.2.144 replica UUID 149243566685142

bucket default, vbucket vb_812 :: Original node 10.6.2.147 active :: seq 0, Change node 10.6.2.144 replica :: seq 10

bucket default, vbucket vb_428 :: Original node 10.6.2.147 replica :: seq 32, Change node 10.6.2.144 active :: seq 0

bucket default, vbucket vb_453 :: Original node 10.6.2.147 replica :: seq 8, Change node 10.6.2.144 active :: seq 0

bucket default, vbucket vb_826 :: Original node 10.6.2.147 active :: seq 32, Change node 10.6.2.144 replica :: seq 0

bucket default, vbucket vb_824 :: Original node 10.6.2.147 active :: UUID 129232167865782, Change node 10.6.2.144 replica UUID 151460545795048

bucket default, vbucket vb_824 :: Original node 10.6.2.147 active :: seq 0, Change node 10.6.2.144 replica :: seq 6

bucket default, vbucket vb_793 :: Original node 10.6.2.147 active :: seq 0, Change node 10.6.2.144 replica :: seq 36

bucket default, vbucket vb_827 :: Original node 10.6.2.147 active :: UUID 35942887977754, Change node 10.6.2.144 replica UUID 158228554855696

bucket default, vbucket vb_828 :: Original node 10.6.2.147 active :: seq 30, Change node 10.6.2.144 replica :: seq 0

bucket default, vbucket vb_829 :: Original node 10.6.2.147 active :: UUID 218293703915510, Change node 10.6.2.144 replica UUID 171958565730256

bucket default, vbucket vb_485 :: Original node 10.6.2.147 replica :: UUID 136236076358264, Change node 10.6.2.144 active UUID 203713664726335

bucket default, vbucket vb_484 :: Original node 10.6.2.147 replica :: seq 32, Change node 10.6.2.144 active :: seq 0

bucket default, vbucket vb_482 :: Original node 10.6.2.147 replica :: UUID 158488793487606, Change node 10.6.2.144 active UUID 82476389453630

bucket default, vbucket vb_796 :: Original node 10.6.2.147 active :: seq 0, Change node 10.6.2.144 replica :: seq 10

bucket default, vbucket vb_805 :: Original node 10.6.2.147 active :: seq 0, Change node 10.6.2.144 replica :: seq 4

bucket default, vbucket vb_450 :: Original node 10.6.2.147 replica :: UUID 205528851843218, Change node 10.6.2.144 active UUID 41653863316282

bucket default, vbucket vb_451 :: Original node 10.6.2.147 replica :: seq 0, Change node 10.6.2.144 active :: seq 8

bucket default, vbucket vb_841 :: Original node 10.6.2.147 active :: seq 6, Change node 10.6.2.144 replica :: seq 0

bucket default, vbucket vb_170 :: Original node 10.6.2.147 replica :: UUID 121352151204808, Change node 10.6.2.145 active UUID 68421505456029

bucket default, vbucket vb_962 :: Original node 10.6.2.147 active :: UUID 40921299218366, Change node 10.6.2.145 replica UUID 60625928695934

bucket default, vbucket vb_133 :: Original node 10.6.2.147 replica :: UUID 100906923536626, Change node 10.6.2.145 active UUID 105627562444235

bucket default, vbucket vb_132 :: Original node 10.6.2.147 replica :: UUID 49850561382592, Change node 10.6.2.145 active UUID 229102890267403

bucket default, vbucket vb_130 :: Original node 10.6.2.147 replica :: seq 0, Change node 10.6.2.145 active :: seq 10

bucket default, vbucket vb_147 :: Original node 10.6.2.147 replica :: UUID 211716012867985, Change node 10.6.2.145 active UUID 39619982790725

bucket default, vbucket vb_140 :: Original node 10.6.2.147 replica :: seq 0, Change node 10.6.2.145 active :: seq 6

bucket default, vbucket vb_148 :: Original node 10.6.2.147 replica :: seq 28, Change node 10.6.2.145 active :: seq 0

bucket default, vbucket vb_102 :: Original node 10.6.2.147 replica :: UUID 271194182420584, Change node 10.6.2.145 active UUID 198256209247406

bucket default, vbucket vb_101 :: Original node 10.6.2.147 replica :: seq 6, Change node 10.6.2.145 active :: seq 0

bucket default, vbucket vb_106 :: Original node 10.6.2.147 replica :: seq 2, Change node 10.6.2.145 active :: seq 0

bucket default, vbucket vb_104 :: Original node 10.6.2.147 replica :: UUID 71419315688224, Change node 10.6.2.145 active UUID 100295122406038

bucket default, vbucket vb_105 :: Original node 10.6.2.147 replica :: UUID 188805885824927, Change node 10.6.2.145 active UUID 213972916959752

bucket default, vbucket vb_1007 :: Original node 10.6.2.147 active :: UUID 78835297695063, Change node 10.6.2.145 replica UUID 92358760256970

bucket default, vbucket vb_1006 :: Original node 10.6.2.147 active :: seq 2, Change node 10.6.2.145 replica :: seq 0

bucket default, vbucket vb_1008 :: Original node 10.6.2.147 active :: UUID 270539579347103, Change node 10.6.2.145 replica UUID 208117369250783

bucket default, vbucket vb_988 :: Original node 10.6.2.147 active :: seq 30, Change node 10.6.2.145 replica :: seq 0

bucket default, vbucket vb_989 :: Original node 10.6.2.147 active :: seq 0, Change node 10.6.2.145 replica :: seq 36

bucket default, vbucket vb_976 :: Original node 10.6.2.147 active :: UUID 137376693802083, Change node 10.6.2.145 replica UUID 66654391520384

bucket default, vbucket vb_974 :: Original node 10.6.2.147 active :: UUID 260712028481736, Change node 10.6.2.145 replica UUID 193146029782058

bucket default, vbucket vb_973 :: Original node 10.6.2.147 active :: seq 4, Change node 10.6.2.145 replica :: seq 0

bucket default, vbucket vb_978 :: Original node 10.6.2.147 active :: UUID 145686233704926, Change node 10.6.2.145 replica UUID 9657818920246

bucket default, vbucket vb_88 :: Original node 10.6.2.147 replica :: UUID 6526210963228, Change node 10.6.2.145 active UUID 9496580064800

bucket default, vbucket vb_947 :: Original node 10.6.2.147 active :: seq 10, Change node 10.6.2.145 replica :: seq 0

bucket default, vbucket vb_942 :: Original node 10.6.2.147 active :: UUID 75111445607199, Change node 10.6.2.145 replica UUID 186646822651977

bucket default, vbucket vb_942 :: Original node 10.6.2.147 active :: seq 8, Change node 10.6.2.145 replica :: seq 0

bucket default, vbucket vb_982 :: Original node 10.6.2.147 active :: seq 0, Change node 10.6.2.145 replica :: seq 6

bucket default, vbucket vb_113 :: Original node 10.6.2.147 replica :: UUID 35422663018472, Change node 10.6.2.145 active UUID 23860153524674

bucket default, vbucket vb_113 :: Original node 10.6.2.147 replica :: seq 10, Change node 10.6.2.145 active :: seq 0

bucket default, vbucket vb_114 :: Original node 10.6.2.147 replica :: UUID 82143415084080, Change node 10.6.2.145 active UUID 67354484706000

bucket default, vbucket vb_1010 :: Original node 10.6.2.147 active :: UUID 269700734338019, Change node 10.6.2.145 replica UUID 79834870266745

bucket default, vbucket vb_1013 :: Original node 10.6.2.147 active :: UUID 176545816124729, Change node 10.6.2.145 replica UUID 161571846010414

bucket default, vbucket vb_990 :: Original node 10.6.2.147 active :: UUID 51677810928801, Change node 10.6.2.145 replica UUID 226061455127364

bucket default, vbucket vb_995 :: Original node 10.6.2.147 active :: UUID 173311840796842, Change node 10.6.2.145 replica UUID 56322691240563

bucket default, vbucket vb_160 :: Original node 10.6.2.147 replica :: UUID 56217035786395, Change node 10.6.2.145 active UUID 211847932266892

bucket default, vbucket vb_168 :: Original node 10.6.2.147 replica :: UUID 112168067328333, Change node 10.6.2.145 active UUID 165254467205992

bucket default, vbucket vb_965 :: Original node 10.6.2.147 active :: UUID 196471036196407, Change node 10.6.2.145 replica UUID 65226901840837

bucket default, vbucket vb_965 :: Original node 10.6.2.147 active :: seq 0, Change node 10.6.2.145 replica :: seq 10

bucket default, vbucket vb_950 :: Original node 10.6.2.147 active :: seq 30, Change node 10.6.2.145 replica :: seq 0

bucket default, vbucket vb_957 :: Original node 10.6.2.147 active :: UUID 256205179248529, Change node 10.6.2.145 replica UUID 92888744811517

bucket default, vbucket vb_956 :: Original node 10.6.2.147 active :: seq 14, Change node 10.6.2.145 replica :: seq 0

bucket default, vbucket vb_952 :: Original node 10.6.2.147 active :: UUID 183075201224989, Change node 10.6.2.145 replica UUID 65096721583493

bucket default, vbucket vb_952 :: Original node 10.6.2.147 active :: seq 0, Change node 10.6.2.145 replica :: seq 34

bucket default, vbucket vb_122 :: Original node 10.6.2.147 replica :: UUID 23077295879957, Change node 10.6.2.145 active UUID 160880366627294

bucket default, vbucket vb_1020 :: Original node 10.6.2.147 active :: UUID 277329679043778, Change node 10.6.2.145 replica UUID 84987331934868