Case:: UUID Changes during graceful-failover-followed-by-addon with recover type = delta replica=1,bucket=default Issue Observed due to Node 10.6.2.150 being failed over and then added-back. vb_28 : {'uuid': 'Expected 162816295095290 :: Actual 126122070037587'} vb_29 : {'uuid': 'Expected 217402359481926 :: Actual 188586946904115'} vb_26 : {'uuid': 'Expected 76205354514325 :: Actual 201466466282756'} vb_27 : {'uuid': 'Expected 10316033053601 :: Actual 129697191560595'} vb_31 : {'uuid': 'Expected 56894599871883 :: Actual 271417789529123'} vb_30 : {'uuid': 'Expected 77677303413706 :: Actual 84498997010846'} Before Graceful-Add-on Operation ----- Bucket default ----- -------------Node 10.6.2.147------------ for vbucket vb_9 :: for key high_seqno = 31 :: for key uuid = 209137107872683 :: for key purge_seqno = 0 for vbucket vb_8 :: for key purge_seqno = 0 :: for key uuid = 249284617261212 :: for key high_seqno = 31 for vbucket vb_1 :: for key purge_seqno = 0 :: for key uuid = 459978698107 :: for key high_seqno = 31 for vbucket vb_0 :: for key purge_seqno = 0 :: for key uuid = 37161762678510 :: for key high_seqno = 32 for vbucket vb_7 :: for key high_seqno = 32 :: for key uuid = 31594799638269 :: for key purge_seqno = 0 for vbucket vb_11 :: for key purge_seqno = 0 :: for key uuid = 255082045131993 :: for key high_seqno = 31 for vbucket vb_10 :: for key high_seqno = 31 :: for key uuid = 174449317499755 :: for key purge_seqno = 0 for vbucket vb_13 :: for key purge_seqno = 0 :: for key uuid = 123129323999006 :: for key high_seqno = 31 for vbucket vb_12 :: for key high_seqno = 31 :: for key uuid = 140006755461460 :: for key purge_seqno = 0 for vbucket vb_15 :: for key purge_seqno = 0 :: for key uuid = 33367807181451 :: for key high_seqno = 31 for vbucket vb_16 :: for key high_seqno = 31 :: for key uuid = 57315532098084 :: for key purge_seqno = 0 for vbucket vb_21 :: for key purge_seqno = 0 :: for key uuid = 143608189005922 :: for key high_seqno = 31 for vbucket vb_28 :: for key high_seqno = 32 :: for key uuid = 162816295095290 :: for key purge_seqno = 0 -------------Node 10.6.2.144------------ for vbucket vb_8 :: for key purge_seqno = 0 :: for key uuid = 249284617261212 :: for key high_seqno = 31 for vbucket vb_1 :: for key purge_seqno = 0 :: for key uuid = 459978698107 :: for key high_seqno = 31 for vbucket vb_0 :: for key purge_seqno = 0 :: for key uuid = 37161762678510 :: for key high_seqno = 32 for vbucket vb_3 :: for key high_seqno = 31 :: for key uuid = 240345536736377 :: for key purge_seqno = 0 for vbucket vb_2 :: for key high_seqno = 32 :: for key uuid = 194543325016073 :: for key purge_seqno = 0 for vbucket vb_5 :: for key high_seqno = 32 :: for key uuid = 12116495437193 :: for key purge_seqno = 0 for vbucket vb_4 :: for key purge_seqno = 0 :: for key uuid = 261100441022801 :: for key high_seqno = 31 for vbucket vb_7 :: for key high_seqno = 32 :: for key uuid = 31594799638269 :: for key purge_seqno = 0 for vbucket vb_6 :: for key purge_seqno = 0 :: for key uuid = 267912971164999 :: for key high_seqno = 31 for vbucket vb_14 :: for key purge_seqno = 0 :: for key uuid = 179829605726349 :: for key high_seqno = 31 for vbucket vb_20 :: for key high_seqno = 31 :: for key uuid = 211824606038749 :: for key purge_seqno = 0 for vbucket vb_26 :: for key purge_seqno = 0 :: for key uuid = 76205354514325 :: for key high_seqno = 31 for vbucket vb_27 :: for key purge_seqno = 0 :: for key uuid = 10316033053601 :: for key high_seqno = 32 -------------Node 10.6.2.150------------ for vbucket vb_5 :: for key high_seqno = 32 :: for key uuid = 12116495437193 :: for key purge_seqno = 0 for vbucket vb_6 :: for key purge_seqno = 0 :: for key uuid = 267912971164999 :: for key high_seqno = 31 for vbucket vb_28 :: for key high_seqno = 32 :: for key uuid = 162816295095290 :: for key purge_seqno = 0 for vbucket vb_29 :: for key purge_seqno = 0 :: for key uuid = 217402359481926 :: for key high_seqno = 31 for vbucket vb_13 :: for key purge_seqno = 0 :: for key uuid = 123129323999006 :: for key high_seqno = 31 for vbucket vb_26 :: for key purge_seqno = 0 :: for key uuid = 76205354514325 :: for key high_seqno = 31 for vbucket vb_19 :: for key purge_seqno = 0 :: for key uuid = 77103211430394 :: for key high_seqno = 31 for vbucket vb_27 :: for key purge_seqno = 0 :: for key uuid = 10316033053601 :: for key high_seqno = 32 for vbucket vb_24 :: for key high_seqno = 31 :: for key uuid = 26739229845145 :: for key purge_seqno = 0 for vbucket vb_25 :: for key high_seqno = 32 :: for key uuid = 146647375057390 :: for key purge_seqno = 0 for vbucket vb_31 :: for key purge_seqno = 0 :: for key uuid = 56894599871883 :: for key high_seqno = 31 for vbucket vb_30 :: for key high_seqno = 32 :: for key uuid = 77677303413706 :: for key purge_seqno = 0 -------------Node 10.6.2.148------------ for vbucket vb_9 :: for key high_seqno = 31 :: for key uuid = 209137107872683 :: for key purge_seqno = 0 for vbucket vb_10 :: for key high_seqno = 31 :: for key uuid = 174449317499755 :: for key purge_seqno = 0 for vbucket vb_3 :: for key high_seqno = 31 :: for key uuid = 240345536736377 :: for key purge_seqno = 0 for vbucket vb_2 :: for key high_seqno = 32 :: for key uuid = 194543325016073 :: for key purge_seqno = 0 for vbucket vb_29 :: for key purge_seqno = 0 :: for key uuid = 217402359481926 :: for key high_seqno = 31 for vbucket vb_15 :: for key purge_seqno = 0 :: for key uuid = 33367807181451 :: for key high_seqno = 31 for vbucket vb_14 :: for key purge_seqno = 0 :: for key uuid = 179829605726349 :: for key high_seqno = 31 for vbucket vb_17 :: for key purge_seqno = 0 :: for key uuid = 116951957067501 :: for key high_seqno = 31 for vbucket vb_16 :: for key high_seqno = 31 :: for key uuid = 57315532098084 :: for key purge_seqno = 0 for vbucket vb_19 :: for key purge_seqno = 0 :: for key uuid = 77103211430394 :: for key high_seqno = 31 for vbucket vb_18 :: for key high_seqno = 31 :: for key uuid = 270175941613828 :: for key purge_seqno = 0 for vbucket vb_22 :: for key high_seqno = 31 :: for key uuid = 270335363769066 :: for key purge_seqno = 0 for vbucket vb_23 :: for key purge_seqno = 0 :: for key uuid = 52714918214896 :: for key high_seqno = 31 -------------Node 10.6.2.149------------ for vbucket vb_4 :: for key purge_seqno = 0 :: for key uuid = 261100441022801 :: for key high_seqno = 31 for vbucket vb_11 :: for key purge_seqno = 0 :: for key uuid = 255082045131993 :: for key high_seqno = 31 for vbucket vb_12 :: for key purge_seqno = 0 :: for key uuid = 140006755461460 :: for key high_seqno = 31 for vbucket vb_17 :: for key purge_seqno = 0 :: for key uuid = 116951957067501 :: for key high_seqno = 31 for vbucket vb_18 :: for key high_seqno = 31 :: for key uuid = 270175941613828 :: for key purge_seqno = 0 for vbucket vb_20 :: for key high_seqno = 31 :: for key uuid = 211824606038749 :: for key purge_seqno = 0 for vbucket vb_21 :: for key purge_seqno = 0 :: for key uuid = 143608189005922 :: for key high_seqno = 31 for vbucket vb_22 :: for key high_seqno = 31 :: for key uuid = 270335363769066 :: for key purge_seqno = 0 for vbucket vb_23 :: for key purge_seqno = 0 :: for key uuid = 52714918214896 :: for key high_seqno = 31 for vbucket vb_24 :: for key high_seqno = 31 :: for key uuid = 26739229845145 :: for key purge_seqno = 0 for vbucket vb_25 :: for key high_seqno = 32 :: for key uuid = 146647375057390 :: for key purge_seqno = 0 for vbucket vb_31 :: for key purge_seqno = 0 :: for key uuid = 56894599871883 :: for key high_seqno = 31 for vbucket vb_30 :: for key high_seqno = 32 :: for key uuid = 77677303413706 :: for key purge_seqno = 0 After Graceful-Add-on Operation ----- Bucket default ----- -------------Node 10.6.2.147------------ for vbucket vb_9 :: for key high_seqno = 31 :: for key uuid = 209137107872683 :: for key purge_seqno = 0 for vbucket vb_8 :: for key purge_seqno = 0 :: for key uuid = 249284617261212 :: for key high_seqno = 31 for vbucket vb_1 :: for key purge_seqno = 0 :: for key uuid = 459978698107 :: for key high_seqno = 31 for vbucket vb_0 :: for key purge_seqno = 0 :: for key uuid = 37161762678510 :: for key high_seqno = 32 for vbucket vb_7 :: for key high_seqno = 32 :: for key uuid = 31594799638269 :: for key purge_seqno = 0 for vbucket vb_11 :: for key purge_seqno = 0 :: for key uuid = 255082045131993 :: for key high_seqno = 31 for vbucket vb_10 :: for key high_seqno = 31 :: for key uuid = 174449317499755 :: for key purge_seqno = 0 for vbucket vb_13 :: for key purge_seqno = 0 :: for key uuid = 123129323999006 :: for key high_seqno = 31 for vbucket vb_12 :: for key high_seqno = 31 :: for key uuid = 140006755461460 :: for key purge_seqno = 0 for vbucket vb_15 :: for key purge_seqno = 0 :: for key uuid = 33367807181451 :: for key high_seqno = 31 for vbucket vb_16 :: for key high_seqno = 31 :: for key uuid = 57315532098084 :: for key purge_seqno = 0 for vbucket vb_21 :: for key purge_seqno = 0 :: for key uuid = 143608189005922 :: for key high_seqno = 31 for vbucket vb_28 :: for key high_seqno = 32 :: for key uuid = 126122070037587 :: for key purge_seqno = 0 -------------Node 10.6.2.144------------ for vbucket vb_8 :: for key purge_seqno = 0 :: for key uuid = 249284617261212 :: for key high_seqno = 31 for vbucket vb_1 :: for key purge_seqno = 0 :: for key uuid = 459978698107 :: for key high_seqno = 31 for vbucket vb_0 :: for key purge_seqno = 0 :: for key uuid = 37161762678510 :: for key high_seqno = 32 for vbucket vb_3 :: for key high_seqno = 31 :: for key uuid = 240345536736377 :: for key purge_seqno = 0 for vbucket vb_2 :: for key high_seqno = 32 :: for key uuid = 194543325016073 :: for key purge_seqno = 0 for vbucket vb_5 :: for key high_seqno = 32 :: for key uuid = 12116495437193 :: for key purge_seqno = 0 for vbucket vb_4 :: for key purge_seqno = 0 :: for key uuid = 261100441022801 :: for key high_seqno = 31 for vbucket vb_7 :: for key high_seqno = 32 :: for key uuid = 31594799638269 :: for key purge_seqno = 0 for vbucket vb_6 :: for key purge_seqno = 0 :: for key uuid = 267912971164999 :: for key high_seqno = 31 for vbucket vb_14 :: for key purge_seqno = 0 :: for key uuid = 179829605726349 :: for key high_seqno = 31 for vbucket vb_20 :: for key high_seqno = 31 :: for key uuid = 211824606038749 :: for key purge_seqno = 0 for vbucket vb_26 :: for key purge_seqno = 0 :: for key uuid = 201466466282756 :: for key high_seqno = 31 for vbucket vb_27 :: for key purge_seqno = 0 :: for key uuid = 129697191560595 :: for key high_seqno = 32 -------------Node 10.6.2.150------------ for vbucket vb_5 :: for key high_seqno = 32 :: for key uuid = 12116495437193 :: for key purge_seqno = 0 for vbucket vb_6 :: for key purge_seqno = 0 :: for key uuid = 267912971164999 :: for key high_seqno = 31 for vbucket vb_28 :: for key high_seqno = 32 :: for key uuid = 126122070037587 :: for key purge_seqno = 0 for vbucket vb_29 :: for key purge_seqno = 0 :: for key uuid = 188586946904115 :: for key high_seqno = 31 for vbucket vb_13 :: for key purge_seqno = 0 :: for key uuid = 123129323999006 :: for key high_seqno = 31 for vbucket vb_26 :: for key purge_seqno = 0 :: for key uuid = 201466466282756 :: for key high_seqno = 31 for vbucket vb_19 :: for key purge_seqno = 0 :: for key uuid = 77103211430394 :: for key high_seqno = 31 for vbucket vb_27 :: for key purge_seqno = 0 :: for key uuid = 129697191560595 :: for key high_seqno = 32 for vbucket vb_24 :: for key high_seqno = 31 :: for key uuid = 26739229845145 :: for key purge_seqno = 0 for vbucket vb_25 :: for key high_seqno = 32 :: for key uuid = 146647375057390 :: for key purge_seqno = 0 for vbucket vb_31 :: for key purge_seqno = 0 :: for key uuid = 271417789529123 :: for key high_seqno = 31 for vbucket vb_30 :: for key high_seqno = 32 :: for key uuid = 84498997010846 :: for key purge_seqno = 0 -------------Node 10.6.2.148------------ for vbucket vb_9 :: for key high_seqno = 31 :: for key uuid = 209137107872683 :: for key purge_seqno = 0 for vbucket vb_10 :: for key high_seqno = 31 :: for key uuid = 174449317499755 :: for key purge_seqno = 0 for vbucket vb_3 :: for key high_seqno = 31 :: for key uuid = 240345536736377 :: for key purge_seqno = 0 for vbucket vb_2 :: for key high_seqno = 32 :: for key uuid = 194543325016073 :: for key purge_seqno = 0 for vbucket vb_29 :: for key purge_seqno = 0 :: for key uuid = 188586946904115 :: for key high_seqno = 31 for vbucket vb_15 :: for key purge_seqno = 0 :: for key uuid = 33367807181451 :: for key high_seqno = 31 for vbucket vb_14 :: for key purge_seqno = 0 :: for key uuid = 179829605726349 :: for key high_seqno = 31 for vbucket vb_17 :: for key purge_seqno = 0 :: for key uuid = 116951957067501 :: for key high_seqno = 31 for vbucket vb_16 :: for key high_seqno = 31 :: for key uuid = 57315532098084 :: for key purge_seqno = 0 for vbucket vb_19 :: for key purge_seqno = 0 :: for key uuid = 77103211430394 :: for key high_seqno = 31 for vbucket vb_18 :: for key high_seqno = 31 :: for key uuid = 270175941613828 :: for key purge_seqno = 0 for vbucket vb_22 :: for key high_seqno = 31 :: for key uuid = 270335363769066 :: for key purge_seqno = 0 for vbucket vb_23 :: for key purge_seqno = 0 :: for key uuid = 52714918214896 :: for key high_seqno = 31 -------------Node 10.6.2.149------------ for vbucket vb_4 :: for key purge_seqno = 0 :: for key uuid = 261100441022801 :: for key high_seqno = 31 for vbucket vb_11 :: for key purge_seqno = 0 :: for key uuid = 255082045131993 :: for key high_seqno = 31 for vbucket vb_12 :: for key purge_seqno = 0 :: for key uuid = 140006755461460 :: for key high_seqno = 31 for vbucket vb_17 :: for key purge_seqno = 0 :: for key uuid = 116951957067501 :: for key high_seqno = 31 for vbucket vb_18 :: for key high_seqno = 31 :: for key uuid = 270175941613828 :: for key purge_seqno = 0 for vbucket vb_20 :: for key high_seqno = 31 :: for key uuid = 211824606038749 :: for key purge_seqno = 0 for vbucket vb_21 :: for key purge_seqno = 0 :: for key uuid = 143608189005922 :: for key high_seqno = 31 for vbucket vb_22 :: for key high_seqno = 31 :: for key uuid = 270335363769066 :: for key purge_seqno = 0 for vbucket vb_23 :: for key purge_seqno = 0 :: for key uuid = 52714918214896 :: for key high_seqno = 31 for vbucket vb_24 :: for key high_seqno = 31 :: for key uuid = 26739229845145 :: for key purge_seqno = 0 for vbucket vb_25 :: for key high_seqno = 32 :: for key uuid = 146647375057390 :: for key purge_seqno = 0 for vbucket vb_31 :: for key purge_seqno = 0 :: for key uuid = 271417789529123 :: for key high_seqno = 31 for vbucket vb_30 :: for key high_seqno = 32 :: for key uuid = 84498997010846 :: for key purge_seqno = 0