MagmaStatsTest disk_usage_recovery is intermittently failing

The test is failing on a large proportion of https://cv.jenkins.couchbase.com/job/unit-simple-test/ runs. This issue was caused by changes in MB-52270.

[ RUN ] MagmaStatsTest.disk_usage_recovery
[13:22:55.761][INFO]: [magma]Initializing with Config:

{"ActivePurgersLimit":1,"BloomFilterAccuracy":0.99,"BloomFilterAccuracyForBottomLevel":0.95,"CheckpointCreationInterval":120000,"CheckpointCreationThreshold":0.2,"CompressionAlgo":"LZ4","DataCompressionAlgo":"Zstd","DumpDebugStats":false,"EnableBlockCache":true,"EnableDirectIO":false,"EnableDirectIOWrite":true,"EnableFastOpen":true,"EnableFileHistograms":true,"EnableGroupCommit":false,"EnableLSMTreeVerification":true,"EnableSSTablePreallocation":true,"EnableUpdateStatusForSet":true,"EnableWAL":false,"EnableWALPreallocation":true,"ExpiryFragThreshold":0.25,"GroupCommitMaxSyncWaitDuration":20,"GroupCommitMaxTransactionCount":20,"GroupName":"default","HeartbeatInterval":0,"IOQueueDepth":1,"KVStorePurgerInterval":60,"KeyTreeAlignBlocks":false,"KeyTreeBlockSize":4096,"KeyTreeIndexBlockSize":4096,"KeyTreeMaxIndexDepth":3,"LSDFragmentationRatio":0.25,"LSDNumLevels":5,"LSMBlockWritesThresholdRatio":8.0,"LSMLevelSizeMultiplier":10,"LSMMaxBaseLevelSize":4194304,"LSMMaxLevel0CompactionTTL":300,"LSMMaxNumLevel0Tables":16,"LSMMaxSSTableSize":2097152,"LSMMinCompactSize":4931584,"LSMNumLevels":5,"LSMNumMemtables":2,"LSMSizeMultiplierBaseLevel":1,"LocalNumLevels":2,"LogLevel":"debug","MaxCheckpoints":10,"MaxCoroutineStackSize":262144,"MaxDeltaSSTableDynamicSizeRatio":0.025,"MaxKVStoreLSDBufferSize":8192,"MaxKVStores":1024,"MaxRecoveryBytes":67108864,"MaxSSTableDynamicSizeRatio":0.05,"MaxWALSize":4294967296,"MaxWriteCacheSize":67108864,"MemoryQuota":671088640,"MinCheckpointCreationInterval":0,"MinCheckpointOverheadSizeThreshold":2097152,"MinValueSize":32,"NumCompactors":1,"NumFileHistogramShards":16,"NumFlushers":1,"Path":"disk_usage_recovery","ReadAheadSize":0,"ReadOnly":false,"SeqTreeAlignBlocks":false,"SeqTreeBlockSize":4096,"SeqTreeIndexBlockSize":4096,"SeqTreeMaxIndexDepth":3,"UseCheckpointCompression":false,"UseCompression":true,"UseStableCheckpoint":false,"WALBufferSize":2097152,"WALNumIORetries":16,"WALSegmentSize":67108864,"WriteCacheRatio":0.2}

[13:22:55.777][INFO]: [magma]Recovering with Config:

{"ActivePurgersLimit":1,"BloomFilterAccuracy":0.99,"BloomFilterAccuracyForBottomLevel":0.95,"CheckpointCreationInterval":120000,"CheckpointCreationThreshold":0.2,"CompressionAlgo":"LZ4","DataCompressionAlgo":"Zstd","DumpDebugStats":false,"EnableBlockCache":true,"EnableDirectIO":false,"EnableDirectIOWrite":true,"EnableFastOpen":true,"EnableFileHistograms":true,"EnableGroupCommit":false,"EnableLSMTreeVerification":true,"EnableSSTablePreallocation":true,"EnableUpdateStatusForSet":true,"EnableWAL":false,"EnableWALPreallocation":true,"ExpiryFragThreshold":0.25,"GroupCommitMaxSyncWaitDuration":20,"GroupCommitMaxTransactionCount":20,"GroupName":"default","HeartbeatInterval":0,"IOQueueDepth":1,"KVStorePurgerInterval":60,"KeyTreeAlignBlocks":false,"KeyTreeBlockSize":4096,"KeyTreeIndexBlockSize":4096,"KeyTreeMaxIndexDepth":3,"LSDFragmentationRatio":0.25,"LSDNumLevels":5,"LSMBlockWritesThresholdRatio":8.0,"LSMLevelSizeMultiplier":10,"LSMMaxBaseLevelSize":4194304,"LSMMaxLevel0CompactionTTL":300,"LSMMaxNumLevel0Tables":16,"LSMMaxSSTableSize":2097152,"LSMMinCompactSize":4931584,"LSMNumLevels":5,"LSMNumMemtables":2,"LSMSizeMultiplierBaseLevel":1,"LocalNumLevels":2,"LogLevel":"debug","MaxCheckpoints":10,"MaxCoroutineStackSize":262144,"MaxDeltaSSTableDynamicSizeRatio":0.025,"MaxKVStoreLSDBufferSize":8192,"MaxKVStores":1024,"MaxRecoveryBytes":67108864,"MaxSSTableDynamicSizeRatio":0.05,"MaxWALSize":4294967296,"MaxWriteCacheSize":67108864,"MemoryQuota":671088640,"MinCheckpointCreationInterval":0,"MinCheckpointOverheadSizeThreshold":2097152,"MinValueSize":32,"NumCompactors":1,"NumFileHistogramShards":16,"NumFlushers":1,"Path":"disk_usage_recovery","ReadAheadSize":0,"ReadOnly":false,"SeqTreeAlignBlocks":false,"SeqTreeBlockSize":4096,"SeqTreeIndexBlockSize":4096,"SeqTreeMaxIndexDepth":3,"UseCheckpointCompression":false,"UseCompression":true,"UseStableCheckpoint":false,"WALBufferSize":2097152,"WALNumIORetries":16,"WALSegmentSize":67108864,"WriteCacheRatio":0.2}

[13:22:55.781][INFO]: [magma/kvstore-1/rev-000000001]KVStore::Open kvstore-1/rev-000000001 highSeqno:0 replayOffset:0:0
[13:22:55.853][INFO]: [magma]Closing magma db (thread pool refcount 1)
[13:22:55.853][INFO]: [magma/kvstore-1/rev-000000001]KVStore::Shutdown
[13:22:55.853][INFO]: [magma]Completed closing magma db
[13:22:55.855][INFO]: [magma]Initializing with Config:

{"ActivePurgersLimit":1,"BloomFilterAccuracy":0.99,"BloomFilterAccuracyForBottomLevel":0.95,"CheckpointCreationInterval":120000,"CheckpointCreationThreshold":0.2,"CompressionAlgo":"LZ4","DataCompressionAlgo":"Zstd","DumpDebugStats":false,"EnableBlockCache":true,"EnableDirectIO":false,"EnableDirectIOWrite":true,"EnableFastOpen":true,"EnableFileHistograms":true,"EnableGroupCommit":false,"EnableLSMTreeVerification":true,"EnableSSTablePreallocation":true,"EnableUpdateStatusForSet":true,"EnableWAL":false,"EnableWALPreallocation":true,"ExpiryFragThreshold":0.25,"GroupCommitMaxSyncWaitDuration":20,"GroupCommitMaxTransactionCount":20,"GroupName":"default","HeartbeatInterval":0,"IOQueueDepth":1,"KVStorePurgerInterval":60,"KeyTreeAlignBlocks":false,"KeyTreeBlockSize":4096,"KeyTreeIndexBlockSize":4096,"KeyTreeMaxIndexDepth":3,"LSDFragmentationRatio":0.25,"LSDNumLevels":5,"LSMBlockWritesThresholdRatio":8.0,"LSMLevelSizeMultiplier":10,"LSMMaxBaseLevelSize":4194304,"LSMMaxLevel0CompactionTTL":300,"LSMMaxNumLevel0Tables":16,"LSMMaxSSTableSize":2097152,"LSMMinCompactSize":4931584,"LSMNumLevels":5,"LSMNumMemtables":2,"LSMSizeMultiplierBaseLevel":1,"LocalNumLevels":2,"LogLevel":"debug","MaxCheckpoints":10,"MaxCoroutineStackSize":262144,"MaxDeltaSSTableDynamicSizeRatio":0.025,"MaxKVStoreLSDBufferSize":8192,"MaxKVStores":1024,"MaxRecoveryBytes":67108864,"MaxSSTableDynamicSizeRatio":0.05,"MaxWALSize":4294967296,"MaxWriteCacheSize":67108864,"MemoryQuota":671088640,"MinCheckpointCreationInterval":0,"MinCheckpointOverheadSizeThreshold":2097152,"MinValueSize":32,"NumCompactors":1,"NumFileHistogramShards":16,"NumFlushers":1,"Path":"disk_usage_recovery","ReadAheadSize":0,"ReadOnly":false,"SeqTreeAlignBlocks":false,"SeqTreeBlockSize":4096,"SeqTreeIndexBlockSize":4096,"SeqTreeMaxIndexDepth":3,"UseCheckpointCompression":false,"UseCompression":true,"UseStableCheckpoint":false,"WALBufferSize":2097152,"WALNumIORetries":16,"WALSegmentSize":67108864,"WriteCacheRatio":0.2}

[13:22:55.856][INFO]: [magma]Recovering with Config:

{"ActivePurgersLimit":1,"BloomFilterAccuracy":0.99,"BloomFilterAccuracyForBottomLevel":0.95,"CheckpointCreationInterval":120000,"CheckpointCreationThreshold":0.2,"CompressionAlgo":"LZ4","DataCompressionAlgo":"Zstd","DumpDebugStats":false,"EnableBlockCache":true,"EnableDirectIO":false,"EnableDirectIOWrite":true,"EnableFastOpen":true,"EnableFileHistograms":true,"EnableGroupCommit":false,"EnableLSMTreeVerification":true,"EnableSSTablePreallocation":true,"EnableUpdateStatusForSet":true,"EnableWAL":false,"EnableWALPreallocation":true,"ExpiryFragThreshold":0.25,"GroupCommitMaxSyncWaitDuration":20,"GroupCommitMaxTransactionCount":20,"GroupName":"default","HeartbeatInterval":0,"IOQueueDepth":1,"KVStorePurgerInterval":60,"KeyTreeAlignBlocks":false,"KeyTreeBlockSize":4096,"KeyTreeIndexBlockSize":4096,"KeyTreeMaxIndexDepth":3,"LSDFragmentationRatio":0.25,"LSDNumLevels":5,"LSMBlockWritesThresholdRatio":8.0,"LSMLevelSizeMultiplier":10,"LSMMaxBaseLevelSize":4194304,"LSMMaxLevel0CompactionTTL":300,"LSMMaxNumLevel0Tables":16,"LSMMaxSSTableSize":2097152,"LSMMinCompactSize":4931584,"LSMNumLevels":5,"LSMNumMemtables":2,"LSMSizeMultiplierBaseLevel":1,"LocalNumLevels":2,"LogLevel":"debug","MaxCheckpoints":10,"MaxCoroutineStackSize":262144,"MaxDeltaSSTableDynamicSizeRatio":0.025,"MaxKVStoreLSDBufferSize":8192,"MaxKVStores":1024,"MaxRecoveryBytes":67108864,"MaxSSTableDynamicSizeRatio":0.05,"MaxWALSize":4294967296,"MaxWriteCacheSize":67108864,"MemoryQuota":671088640,"MinCheckpointCreationInterval":0,"MinCheckpointOverheadSizeThreshold":2097152,"MinValueSize":32,"NumCompactors":1,"NumFileHistogramShards":16,"NumFlushers":1,"Path":"disk_usage_recovery","ReadAheadSize":0,"ReadOnly":false,"SeqTreeAlignBlocks":false,"SeqTreeBlockSize":4096,"SeqTreeIndexBlockSize":4096,"SeqTreeMaxIndexDepth":3,"UseCheckpointCompression":false,"UseCompression":true,"UseStableCheckpoint":false,"WALBufferSize":2097152,"WALNumIORetries":16,"WALSegmentSize":67108864,"WriteCacheRatio":0.2}

[13:22:55.857][INFO]: [magma/kvstore-1/rev-000000001]KVStore::Open kvstore-1/rev-000000001 highSeqno:1000 replayOffset:0:0
/home/couchbase/jenkins/workspace/unit-simple-test/magma/tests/stats_test.cc:122: Failure
Expected equality of these values:
stats.TotalDiskUsage
Which is: 311296
du
Which is: 319488