[Magma-CDC] Magma fragmentation exceeds the configured value(for few shards it was >90% and remains in the same state)

 rw_7:magma:                   {"ActiveBloomFilterSize":243789,"ActiveDataSize":12149494,"ActiveDiskUsage":258752512,"ActiveIndexBlocksSize":1796541,"BlockCacheHitRatio":98.756591796875,"BlockCacheHits":1035376,"BlockCacheHitsPerSec":0.0,"BlockCacheMemUsed":6193600,"BlockCacheMisses":13036,"BlockCacheMissesPerSec":0.0,"BlockCacheQuota":9865680,"BloomFilterFPR":0.017101839184761047,"BloomFilterMemUsedRatio":0.0002462599950376898,"BloomFilterMemoryQuota":998034633,"BufferMemUsed":8960,"BytesIncoming":671342241,"BytesIncomingPerSec":0.0,"BytesOutgoing":0,"BytesOutgoingPerSec":0.0,"BytesPerRead":7.357446193695068,"CheckpointOverhead":0,"CheckpointOverheadPeriod":600,"CheckpointOverheadRatio":0.0,"CompactQueueSize":0,"DataBlocksCompressSize":244491825,"DataBlocksCompressionRatio":1.4256625962851723,"DataBlocksSize":348562850,"DataBlocksSpaceReductionEstimate":29.85717640305041,"FlushQueueSize":8,"Fragmentation":0.9627501368522644,"FragmentationWithHistory":0.9530458450317383,"FragmentationWithOverhead":0.9631026983261108,"HistogramMemUsed":101776,"HistoryDiskUsage":17633280,"HistoryLogicalDataSize":4275948,"HistoryLogicalDiskSize":26178908,"HistorySizeBytesEvicted":430174208,"HistoryTimeBytesEvicted":32088064,"IndexResidentRatio":3.6382603645324707,"KeyTreeDataBlockSize":4096,"KeyTreeIndexBlockSize":4096,"LSMTreeObjectMemUsed":38864,"LogicalDataSize":15970861,"LogicalDiskSize":340137617,"ManagedFileObjectMemUsed":1824,"MemoryQuota":1247543296,"MemoryQuotaLowWaterMark":249508662,"NActivePurgers":0,"NBlocksCached":1752,"NBloomFilterHits":2315597,"NBloomFilterHitsPerSec":0.0,"NBloomFilterMisses":40290,"NBloomFilterMissesPerSec":0.0,"NCommitBatches":20957,"NCommitBatchesPerSec":0.0,"NCompacts":24,"NCompactsPerSec":0.0,"NDataLevelFileCountCompacts":0,"NDeletes":0,"NDeletesPerSec":0.0,"NFileCountCompacts":0,"NFlushes":63,"NFlushesPerSec":0.0,"NGetStats":1480,"NGetStatsComputed":1480,"NGetStatsComputedPerSec":0.7333333492279053,"NGetStatsPerSec":0.7333333492279053,"NGets":312500,"NGetsPerSec":0.0,"NInserts":0,"NInsertsPerSec":0.0,"NInternalFragmentationCompacts":0,"NPurgerRuns":0,"NReadBytes":331471364,"NReadBytesCompact":318747944,"NReadBytesCompactPerSec":0.0,"NReadBytesGet":12723420,"NReadBytesGetPerSec":0.0,"NReadBytesPerSec":0.0,"NReadBytesSet":10424218,"NReadBytesSetPerSec":0.0,"NReadIOs":109273,"NReadIOsGet":4875,"NReadIOsGetPerSec":0.0,"NReadIOsPerSec":0.0,"NReadIOsSet":4167,"NReadIOsSetPerSec":0.0,"NRetryCompacts":4,"NSets":625002,"NSetsPerSec":0.0,"NSyncs":156,"NSyncsPerSec":0.0,"NTTLCompacts":2,"NTableFiles":20,"NTablesCreated":175,"NTablesDeleted":151,"NWriteBytes":1331869093,"NWriteBytesCompact":65941504,"NWriteBytesCompactPerSec":0.0,"NWriteBytesPerSec":0.0,"NWriteIOs":23052,"NWriteIOsPerSec":0.0,"NWriterCompacts":0,"NonResidentBloomFilterSize":0,"NumKVStores":2,"NumNormalFlushes":11,"NumPersistentFlushes":52,"NumSyncFlushes":0,"ReadAheadBufferMemUsed":0,"ReadAmp":0.0,"ReadAmpGet":0.0,"ReadIOAmp":0.002265600021928549,"ReadIOAmpSet":0.0,"RecentBlockCacheHitRatio":0.0,"RecentBloomFilterFPR":0.0,"RecentBytesPerRead":0.0,"RecentReadAmp":0.0,"RecentReadAmpGet":0.0,"RecentReadIOAmp":0.0,"RecentReadIOAmpSet":0.0,"RecentWriteAmp":0.0,"ReplayWALOffset":"47:2101248","SeqTreeDataBlockSize":4096,"SeqTreeIndexBlockSize":4096,"TableMetaMemUsed":6156,"TableObjectMemUsed":3800,"TotalBloomFilterMemUsed":245776,"TotalBloomFilterSize":243789,"TotalDiskUsage":258752512,"TotalIndexBlocksSize":1796541,"TotalMemUsed":6778652,"TreeSnapshotMemoryUsed":19680,"WALBufferMemUsed":81920,"WALMemUsed":147536,"WriteAmp":1.9838896989822388,"WriteCacheMemUsed":5040,"WriteCacheQuota":134217728,"keyStats":{"ActiveBloomFilterSize":243717,"ActiveDiskUsage":8163328,"ActiveIndexBlocksSize":94073,"BloomFilterFPR":0.01710183892521161,"BufferMemUsed":0,"BytesIncoming":31875088,"BytesOutgoing":0,"CheckpointOverhead":0,"CheckpointOverheadPeriod":600,"CompactReader":{"NBytesOutgoing":0,"NDataBlockReads":12026,"NIndexBlockReads":139,"NReadBytes":30383251,"NReadIOs":7987},"CompactWriter":{"NBytesIncoming":39450651,"NDataBlockWrites":8233,"NIndexBlockWrites":81,"NInserts":653612,"NSyncs":0,"NWriteBytes":16953344,"NWriteIOs":39},"HistoryDiskUsage":0,"HistoryLogicalDataSize":0,"HistoryLogicalDiskSize":0,"HistorySizeBytesEvicted":0,"HistoryStartTimestamp":0,"HistoryTimeBytesEvicted":0,"IterateReader":{"NBytesOutgoing":0,"NDataBlockReads":0,"NIndexBlockReads":0,"NReadBytes":0,"NReadIOs":0},"IteratorMemUsed":0,"LevelStats":[{"CompactingSize":0,"DataBlocksCompressSize":0,"DataBlocksCompressionRatio":0.0,"DataBlocksSize":0,"DataBlocksSpaceReductionEstimate":0.0,"HistoryLogicalSize":0,"HistoryPhysicalSize":0,"Level":0,"LogicalSize":0,"NWriteBytes":33697792,"NumTables":0,"PhysicalSize":0,"SizeRatio":0.0,"TargetSize":8388608,"WastedSpace":0},{"CompactingSize":0,"DataBlocksCompressSize":0,"DataBlocksCompressionRatio":0.0,"DataBlocksSize":0,"DataBlocksSpaceReductionEstimate":0.0,"HistoryLogicalSize":0,"HistoryPhysicalSize":0,"Level":1,"LogicalSize":0,"NWriteBytes":0,"NumTables":0,"PhysicalSize":0,"SizeRatio":0.0,"TargetSize":8388608,"WastedSpace":0},{"CompactingSize":0,"DataBlocksCompressSize":0,"DataBlocksCompressionRatio":0.0,"DataBlocksSize":0,"DataBlocksSpaceReductionEstimate":0.0,"HistoryLogicalSize":0,"HistoryPhysicalSize":0,"Level":2,"LogicalSize":0,"NWriteBytes":0,"NumTables":0,"PhysicalSize":0,"SizeRatio":0.0,"TargetSize":8388608,"WastedSpace":0},{"CompactingSize":0,"DataBlocksCompressSize":7762001,"DataBlocksCompressionRatio":2.1665158249786365,"DataBlocksSize":16816498,"DataBlocksSpaceReductionEstimate":53.842940426716666,"HistoryLogicalSize":0,"HistoryPhysicalSize":0,"Level":3,"LogicalSize":8134123,"NWriteBytes":16953344,"NumTables":4,"PhysicalSize":8163328,"SizeRatio":0.9696630239486694,"TargetSize":8388608,"WastedSpace":29205},{"CompactingSize":0,"DataBlocksCompressSize":0,"DataBlocksCompressionRatio":0.0,"DataBlocksSize":0,"DataBlocksSpaceReductionEstimate":0.0,"HistoryLogicalSize":0,"HistoryPhysicalSize":0,"Level":4,"LogicalSize":0,"NWriteBytes":0,"NumTables":0,"PhysicalSize":0,"SizeRatio":0.0,"TargetSize":8388608,"WastedSpace":0}],"LogicalDataSize":0,"LogicalDiskSize":0,"ManagedFileObjectMemUsed":384,"MemtablesMemUsed":1680,"NBloomFilterHits":2315597,"NBloomFilterMisses":40290,"NCompacts":5,"NDataLevelFileCountCompacts":0,"NFailedCompacts":0,"NFileCountCompacts":0,"NInternalFragmentationCompacts":0,"NItemsDel":0,"NItemsDropped":0,"NItemsGarbageCollected":312500,"NItemsGet":937502,"NItemsMerged":0,"NItemsPut":625002,"NItemsTransformed":653612,"NNonL0LocalCompacts":0,"NReadBytes":43106671,"NReadIOs":12862,"NRetryCompacts":0,"NSyncs":52,"NTTLCompacts":0,"NTableFiles":4,"NTablesCreated":34,"NTablesDeleted":30,"NWriteBytes":50675192,"NWriteIOs":165,"NWriterCompacts":0,"NonResidentBloomFilterSize":0,"PendingCheckpointBytes":0,"ReadAheadBufferMemUsed":0,"ReadAmp":0.0,"ReadIOAmp":0.005199988906690332,"Reader":{"NBytesOutgoing":0,"NDataBlockReads":352790,"NIndexBlockReads":683283,"NReadBytes":12723420,"NReadIOs":4875},"TableMetaMemUsed":1654,"TableObjectMemUsed":800,"TotalBloomFilterMemUsed":245760,"TotalBloomFilterSize":243717,"TotalDiskUsage":8163328,"TotalIndexBlocksSize":94073,"TreeSnapshotMemoryUsed":6720,"UID":"","WriteAmp":1.5898055559877984,"Writer":{"NBytesIncoming":37500106,"NDataBlockWrites":7843,"NIndexBlockWrites":96,"NInserts":625002,"NSyncs":52,"NWriteBytes":33721848,"NWriteIOs":126}},"localStats":{"ActiveBloomFilterSize":72,"ActiveDiskUsage":16384,"ActiveIndexBlocksSize":116,"BloomFilterFPR":0.0,"BufferMemUsed":0,"BytesIncoming":11341923,"BytesOutgoing":0,"CheckpointOverhead":0,"CheckpointOverheadPeriod":600,"CompactReader":{"NBytesOutgoing":0,"NDataBlockReads":24,"NIndexBlockReads":24,"NReadBytes":15880,"NReadIOs":48},"CompactWriter":{"NBytesIncoming":1450,"NDataBlockWrites":2,"NIndexBlockWrites":2,"NInserts":8,"NSyncs":0,"NWriteBytes":16384,"NWriteIOs":2},"HistoryDiskUsage":0,"HistoryLogicalDataSize":0,"HistoryLogicalDiskSize":0,"HistorySizeBytesEvicted":0,"HistoryStartTimestamp":0,"HistoryTimeBytesEvicted":0,"IterateReader":{"NBytesOutgoing":0,"NDataBlockReads":0,"NIndexBlockReads":0,"NReadBytes":0,"NReadIOs":0},"IteratorMemUsed":0,"LevelStats":[{"CompactingSize":0,"DataBlocksCompressSize":0,"DataBlocksCompressionRatio":0.0,"DataBlocksSize":0,"DataBlocksSpaceReductionEstimate":0.0,"HistoryLogicalSize":0,"HistoryPhysicalSize":0,"Level":0,"LogicalSize":0,"NWriteBytes":196608,"NumTables":0,"PhysicalSize":0,"SizeRatio":0.0,"TargetSize":8388608,"WastedSpace":0},{"CompactingSize":0,"DataBlocksCompressSize":0,"DataBlocksCompressionRatio":0.0,"DataBlocksSize":0,"DataBlocksSpaceReductionEstimate":0.0,"HistoryLogicalSize":0,"HistoryPhysicalSize":0,"Level":1,"LogicalSize":0,"NWriteBytes":0,"NumTables":0,"PhysicalSize":0,"SizeRatio":0.0,"TargetSize":8388608,"WastedSpace":0},{"CompactingSize":0,"DataBlocksCompressSize":1155,"DataBlocksCompressionRatio":1.309090909090909,"DataBlocksSize":1512,"DataBlocksSpaceReductionEstimate":23.61111111111111,"HistoryLogicalSize":0,"HistoryPhysicalSize":0,"Level":2,"LogicalSize":18283,"NWriteBytes":16384,"NumTables":2,"PhysicalSize":16384,"SizeRatio":0.0021795034408569336,"TargetSize":8388608,"WastedSpace":0}],"LogicalDataSize":18283,"LogicalDiskSize":18283,"ManagedFileObjectMemUsed":192,"MemtablesMemUsed":1680,"NBloomFilterHits":0,"NBloomFilterMisses":0,"NCompacts":2,"NDataLevelFileCountCompacts":0,"NFailedCompacts":0,"NFileCountCompacts":0,"NInternalFragmentationCompacts":0,"NItemsDel":0,"NItemsDropped":0,"NItemsGarbageCollected":41910,"NItemsGet":12,"NItemsMerged":0,"NItemsPut":41918,"NItemsTransformed":0,"NNonL0LocalCompacts":0,"NReadBytes":15880,"NReadIOs":48,"NRetryCompacts":0,"NSyncs":52,"NTTLCompacts":2,"NTableFiles":2,"NTablesCreated":26,"NTablesDeleted":24,"NWriteBytes":235578,"NWriteIOs":78,"NWriterCompacts":0,"NonResidentBloomFilterSize":0,"PendingCheckpointBytes":0,"ReadAheadBufferMemUsed":0,"ReadAmp":0.0,"ReadIOAmp":0.0,"Reader":{"NBytesOutgoing":0,"NDataBlockReads":0,"NIndexBlockReads":0,"NReadBytes":0,"NReadIOs":0},"TableMetaMemUsed":628,"TableObjectMemUsed":400,"TotalBloomFilterMemUsed":16,"TotalBloomFilterSize":72,"TotalDiskUsage":16384,"TotalIndexBlocksSize":116,"TreeSnapshotMemoryUsed":5376,"UID":"","WriteAmp":0.02077055187202382,"Writer":{"NBytesIncoming":13742,"NDataBlockWrites":24,"NIndexBlockWrites":24,"NInserts":52,"NSyncs":52,"NWriteBytes":219194,"NWriteIOs":76}},"seqStats":{"ActiveBloomFilterSize":0,"ActiveDiskUsage":250572800,"ActiveIndexBlocksSize":1702352,"BloomFilterFPR":0.0,"BufferMemUsed":8960,"BytesIncoming":671562842,"BytesOutgoing":0,"CheckpointOverhead":0,"CheckpointOverheadPeriod":600,"CompactReader":{"NBytesOutgoing":0,"NDataBlockReads":95584,"NIndexBlockReads":779,"NReadBytes":288348813,"NReadIOs":96363},"CompactWriter":{"NBytesIncoming":69372748,"NDataBlockWrites":16641,"NIndexBlockWrites":206,"NInserts":186229,"NSyncs":0,"NWriteBytes":48971776,"NWriteIOs":173},"HistoryDiskUsage":17633280,"HistoryLogicalDataSize":4275948,"HistoryLogicalDiskSize":26178908,"HistorySizeBytesEvicted":430174208,"HistoryStartTimestamp":1682321816,"HistoryTimeBytesEvicted":32088064,"IterateReader":{"NBytesOutgoing":0,"NDataBlockReads":0,"NIndexBlockReads":0,"NReadBytes":0,"NReadIOs":0},"IteratorMemUsed":0,"LevelStats":[{"CompactingSize":0,"DataBlocksCompressSize":0,"DataBlocksCompressionRatio":0.0,"DataBlocksSize":0,"DataBlocksSpaceReductionEstimate":0.0,"HistoryLogicalSize":0,"HistoryPhysicalSize":0,"Level":0,"LogicalSize":0,"NWriteBytes":6905856,"NumTables":0,"PhysicalSize":0,"SizeRatio":0.0,"TargetSize":8388608,"WastedSpace":0},{"CompactingSize":0,"DataBlocksCompressSize":0,"DataBlocksCompressionRatio":0.0,"DataBlocksSize":0,"DataBlocksSpaceReductionEstimate":0.0,"HistoryLogicalSize":0,"HistoryPhysicalSize":0,"Level":1,"LogicalSize":0,"NWriteBytes":0,"NumTables":0,"PhysicalSize":0,"SizeRatio":0.0,"TargetSize":8388608,"WastedSpace":0},{"CompactingSize":0,"DataBlocksCompressSize":0,"DataBlocksCompressionRatio":0.0,"DataBlocksSize":0,"DataBlocksSpaceReductionEstimate":0.0,"HistoryLogicalSize":0,"HistoryPhysicalSize":0,"Level":2,"LogicalSize":0,"NWriteBytes":6905856,"NumTables":0,"PhysicalSize":0,"SizeRatio":0.0,"TargetSize":8388608,"WastedSpace":0},{"CompactingSize":0,"DataBlocksCompressSize":3406879,"DataBlocksCompressionRatio":1.0,"DataBlocksSize":3406879,"DataBlocksSpaceReductionEstimate":0.0,"HistoryLogicalSize":324166756,"HistoryPhysicalSize":0,"Level":3,"LogicalSize":324166756,"NWriteBytes":8015872,"NumTables":1,"PhysicalSize":3444736,"SizeRatio":0.0,"TargetSize":170068808,"WastedSpace":4944},{"CompactingSize":0,"DataBlocksCompressSize":244491825,"DataBlocksCompressionRatio":1.4256625962851723,"DataBlocksSize":348562850,"DataBlocksSpaceReductionEstimate":29.85717640305041,"HistoryLogicalSize":26178908,"HistoryPhysicalSize":17633280,"Level":4,"LogicalSize":340137617,"NWriteBytes":531070976,"NumTables":12,"PhysicalSize":247128064,"SizeRatio":0.46268222195877634,"TargetSize":678562292,"WastedSpace":856471}],"LogicalDataSize":15970861,"LogicalDiskSize":340137617,"ManagedFileObjectMemUsed":1248,"MemtablesMemUsed":1680,"NBloomFilterHits":0,"NBloomFilterMisses":0,"NCompacts":17,"NDataLevelFileCountCompacts":0,"NFailedCompacts":0,"NFileCountCompacts":0,"NInternalFragmentationCompacts":0,"NItemsDel":0,"NItemsDropped":0,"NItemsGarbageCollected":156220,"NItemsGet":0,"NItemsMerged":156220,"NItemsPut":937502,"NItemsTransformed":0,"NNonL0LocalCompacts":0,"NReadBytes":288348813,"NReadIOs":96363,"NRetryCompacts":4,"NSyncs":52,"NTTLCompacts":0,"NTableFiles":14,"NTablesCreated":115,"NTablesDeleted":97,"NWriteBytes":539574131,"NWriteIOs":1184,"NWriterCompacts":0,"NonResidentBloomFilterSize":0,"PendingCheckpointBytes":0,"ReadAheadBufferMemUsed":0,"ReadAmp":0.0,"ReadIOAmp":0.0,"Reader":{"NBytesOutgoing":0,"NDataBlockReads":0,"NIndexBlockReads":0,"NReadBytes":0,"NReadIOs":0},"TableMetaMemUsed":3874,"TableObjectMemUsed":2600,"TotalBloomFilterMemUsed":0,"TotalBloomFilterSize":0,"TotalDiskUsage":250572800,"TotalIndexBlocksSize":1702352,"TreeSnapshotMemoryUsed":7584,"UID":"","WriteAmp":0.8034603722163651,"Writer":{"NBytesIncoming":680000360,"NDataBlockWrites":162975,"NIndexBlockWrites":1198,"NInserts":937502,"NSyncs":52,"NWriteBytes":490602355,"NWriteIOs":1011}},"walStats":{"DiskUsed":2101248,"NActiveSegments":1,"NInserts":729791,"NLoggedBytes":741384192,"NSegments":47,"NSyncs":20957,"NTxns":20957,"NWriteBytes":741384192,"NWrites":21625,"PreallocateSupported":true,"TxnSizeEstimate":12567}}

guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/temp_vol.ini bucket_storage=magma,GROUP=vb64;P0,bucket_eviction_policy=fullEviction,randomize_value=True,rerun=false,replicas=0,test_itr=1,enable_dp=false,get-cbcollect-info=True,autoCompactionDefined=true,bucket_history_retention_seconds=600,bucket_history_retention_bytes=3000000000,upgrade_version=7.2.0-5312 -t storage.magma.magma_deletes.BasicDeleteTests.test_create_delete_n_times,num_items=10000000,nodes_init=4,sdk_timeout=60,vbuckets=64,test_itr=1,skip_cleanup=True,GROUP=vb64;P0'