Snapshot start seqno can be non Monotonic in the checkpoint manager

2022-02-27T23:11:00.405845-08:00 CRITICAL (source_op) ActiveStream::processItems: Seen nextSnapStart go backwards vb:257 e.what():'Monotonic<m> (ActiveStream eq_dcpq:eventing:0x3iywM0-700:eventing:0x3iywM0-698:1551649086_172.23.104.17:11210_worker_source_op/scope_0/sbm1_0_0_172.23.104.21:8096/0 (vb:257) nextSnapStart) invariant failed: new value (52781) breaks invariant on current value (52787)' checkpoint state:CheckpointManager[0x7f63a66d0800] with numItems:513 checkpoints:3

    Checkpoint[0x7f63b3989400] with id:2 seqno:{52787,53283} snap:{52787,53283, visible:53283} state:Closed numCursors:2 type:Memory hcs:none  items:[

	{52787,empty,cid:0x1:empty,126,[m]}

	{52787,checkpoint_start,cid:0x1:checkpoint_start,137,[m]}

	{52787,mutation[d],cid:0xb:34BD6C77-90_389774_sbm,247,}

...

	{53283,mutation,cid:0xb:34BD6C77-90_455841_sbm,1550,}

	{53284,set_vbucket_state,cid:0x1:set_vbucket_state,216,[m]}

	{53284,set_vbucket_state,cid:0x1:set_vbucket_state,216,[m]} 

	{53284,checkpoint_end,cid:0x1:checkpoint_end,135,[m]}

]

    Checkpoint[0x7f63b37ba800] with id:3 seqno:{53284,53285} snap:{52781,53285, visible:53285} state:Closed numCursors:0 type:Memory hcs:none  items:[

	{53284,empty,cid:0x1:empty,126,[m]}

	{53284,checkpoint_start,cid:0x1:checkpoint_start,137,[m]}

	{53284,mutation,cid:0xb:224C956B-93_456009,1454,}

	{53285,mutation,cid:0xc:1020F9D2-101_455491_sbm,1320,}

	{53286,checkpoint_end,cid:0x1:checkpoint_end,135,[m]}

]

    Checkpoint[0x7f63a66d0c00] with id:4 seqno:{53286,53290} snap:{53286,53290, visible:53290} state:Open numCursors:2 type:Memory hcs:none  items:[

	{53286,empty,cid:0x1:empty,126,[m]}

	{53286,checkpoint_start,cid:0x1:checkpoint_start,137,[m]}

	{53286,set_vbucket_state,cid:0x1:set_vbucket_state,215,[m]} 

	{53286,mutation,cid:0xc:34BD6C77-90_456040_sbm,1320,}

	{53287,set_vbucket_state,cid:0x1:set_vbucket_state,284,[m]}

	{53287,mutation,cid:0xc:D5D7D46D-102_455534,1223,}

	{53288,mutation,cid:0xc:1020F9D2-101_455559,1454,}

	{53289,mutation,cid:0xc:3DAF36FD-92_456099,1453,}

	{53290,mutation,cid:0xc:3DAF36FD-92_456109,1453,}

]

    cursors:[

        eq_dcpq:eventing:0x3iywM0-700:eventing:0x3iywM0-698:1551649086_172.23.104.17:11210_worker_source_op/scope_0/sbm1_0_0_172.23.104.21:8096/0: CheckpointCursor[0x7f63837b81d0] with name:eq_dcpq:eventing:0x3iywM0-700:eventing:0x3iywM0-698:1551649086_172.23.104.17:11210_worker_source_op/scope_0/sbm1_0_0_172.23.104.21:8096/0 currentCkpt:{id:4 state:Open} currentSeq:53290 distance:8

        eq_dcpq:eventing:0x3iywM0-701:eventing:0x3iywM0-699:1898432783_172.23.104.17:11210_worker_source_op/scope_0/sbm2_0_0_172.23.104.21:8096/0: CheckpointCursor[0x7f63859c0630] with name:eq_dcpq:eventing:0x3iywM0-701:eventing:0x3iywM0-699:1898432783_172.23.104.17:11210_worker_source_op/scope_0/sbm2_0_0_172.23.104.21:8096/0 currentCkpt:{id:2 state:Closed} currentSeq:53283 distance:498

        eq_dcpq:replication:ns_1@172.23.104.17->ns_1@172.23.104.16:source_op: CheckpointCursor[0x7f638622a530] with name:eq_dcpq:replication:ns_1@172.23.104.17->ns_1@172.23.104.16:source_op currentCkpt:{id:4 state:Open} currentSeq:53290 distance:8

        persistence: CheckpointCursor[0x7f637f516bd0] with name:persistence currentCkpt:{id:2 state:Closed} currentSeq:53283 distance:498

    ]