Loading...

Details

Type: Bug
Resolution: Fixed
Priority: Minor
Fix Version/s: 7.2.1
Affects Version/s: 7.2.0
Component/s: storage-engine
Labels:
Environment:
7.2.0-5321

Triage:
Untriaged
Link to Log File, atop/blg, CBCollectInfo, Core dump:
https://cb-engineering.s3.amazonaws.com/new.tar.gz
Story Points:
0
Is this a Regression?:
Unknown

Description

Create a 4 node cluster
Create a magma buckets with (bucket_history_retention_seconds=600,bucket_history_retention_bytes=3000000000,)
Fragmentation value is configured to 20%(Not 50%)
Create 7000000 docs(durability=Majority,
Upsert all the items twice
Delete half of the items
At the end of above steps observed Magma fragmentation for few shards is more than 20% test keeps checking for 300 seconds before asserting for fragmentation, observed for few shards frag value was more than 40%, after 300 seconds though fragmentation starts to drop. waited 20 more mins but for few shards still it was more than 25%, Since it took very long to drop fragmentation, it will be good if dev team can take a look once, One of the shard on Node:172.23.108.186 had fragmentation ~ 29 %) .

Below stats are for node 172.23.108.186 (shard 7)
Magma stats:

 rw_6:magma:                   {"ActiveBloomFilterSize":402232,"ActiveDataSize":366521216,"ActiveDiskUsage":524099584,"ActiveIndexBlocksSize":3628928,"BlockCacheHitRatio":98.33246612548828,"BlockCacheHits":4376005,"BlockCacheHitsPerSec":46.56666564941406,"BlockCacheMemUsed":3652088,"BlockCacheMisses":74209,"BlockCacheMissesPerSec":163.03334045410156,"BlockCacheQuota":17602392,"BloomFilterFPR":0.013477307744324207,"BloomFilterMemUsedRatio":0.0008992890943773091,"BloomFilterMemoryQuota":499069745,"BufferMemUsed":3486016,"BytesIncoming":4770448446,"BytesIncomingPerSec":0.0,"BytesOutgoing":0,"BytesOutgoingPerSec":0.0,"BytesPerRead":16.47450828552246,"CheckpointOverhead":90849280,"CheckpointOverheadPeriod":183,"CheckpointOverheadRatio":0.17334353923797607,"CompactQueueSize":224,"DataBlocksCompressSize":499959487,"DataBlocksCompressionRatio":1.4078367913838588,"DataBlocksSize":703861360,"DataBlocksSpaceReductionEstimate":28.96903915850701,"FlushQueueSize":40,"Fragmentation":0.29789721965789795,"FragmentationWithHistory":0.3006649315357208,"FragmentationWithOverhead":0.500815212726593,"HistogramMemUsed":101776,"HistoryDiskUsage":36577280,"HistoryLogicalDataSize":35159034,"HistoryLogicalDiskSize":52798312,"HistorySizeBytesEvicted":2581094400,"HistoryTimeBytesEvicted":867528704,"IndexResidentRatio":0.5137509703636169,"KeyTreeDataBlockSize":4096,"KeyTreeIndexBlockSize":4096,"LSMTreeObjectMemUsed":136024,"LogicalDataSize":482815330,"LogicalDiskSize":690392009,"ManagedFileObjectMemUsed":46560,"MemoryQuota":623837184,"MemoryQuotaLowWaterMark":124767438,"NActivePurgers":0,"NBlocksCached":1497,"NBloomFilterHits":14895003,"NBloomFilterHitsPerSec":0.0,"NBloomFilterMisses":203487,"NBloomFilterMissesPerSec":0.0,"NCommitBatches":12022,"NCommitBatchesPerSec":0.0,"NCompacts":1800,"NCompactsPerSec":0.699999988079071,"NDataLevelFileCountCompacts":0,"NDeletes":0,"NDeletesPerSec":0.0,"NFileCountCompacts":14,"NFlushes":565,"NFlushesPerSec":0.3333333432674408,"NGetStats":4121,"NGetStatsComputed":4121,"NGetStatsComputedPerSec":1.1333333253860474,"NGetStatsPerSec":1.1333333253860474,"NGets":218985,"NGetsPerSec":0.0,"NInserts":0,"NInsertsPerSec":0.0,"NInternalFragmentationCompacts":4,"NPurgerRuns":7,"NReadBytes":5252511370,"NReadBytesCompact":5180912296,"NReadBytesCompactPerSec":1896466.875,"NReadBytesGet":71599074,"NReadBytesGetPerSec":84.43333435058594,"NReadBytesPerSec":1896551.5,"NReadBytesSet":67991404,"NReadBytesSetPerSec":0.0,"NReadIOs":1731887,"NReadIOsGet":25803,"NReadIOsGetPerSec":0.2666666805744171,"NReadIOsPerSec":631.0,"NReadIOsSet":24878,"NReadIOsSetPerSec":0.0,"NRetryCompacts":1291,"NSets":2679925,"NSetsPerSec":0.0,"NSyncs":1281,"NSyncsPerSec":1.0,"NTTLCompacts":8,"NTableFiles":493,"NTablesCreated":5534,"NTablesDeleted":5150,"NWriteBytes":10716392940,"NWriteBytesCompact":2096816128,"NWriteBytesCompactPerSec":676386.125,"NWriteBytesPerSec":719209.9375,"NWriteIOs":35154,"NWriteIOsPerSec":2.700000047683716,"NWriterCompacts":0,"NonHistoryLogicalDataSize":447656296,"NonHistoryLogicalDiskSize":637593697,"NonResidentBloomFilterSize":447120,"NumKVStores":7,"NumNormalFlushes":132,"NumPersistentFlushes":433,"NumSyncFlushes":0,"ReadAheadBufferMemUsed":0,"ReadAmp":0.0,"ReadAmpGet":0.0,"ReadIOAmp":0.004224033560603857,"ReadIOAmpSet":0.0,"RecentBlockCacheHitRatio":22.216920852661133,"RecentBloomFilterFPR":0.0,"RecentBytesPerRead":0.0,"RecentReadAmp":0.0,"RecentReadAmpGet":0.0,"RecentReadIOAmp":0.0,"RecentReadIOAmpSet":0.0,"RecentWriteAmp":0.0,"ReplayWALOffset":"164:5992448","SeqTreeDataBlockSize":4096,"SeqTreeIndexBlockSize":4096,"TableMetaMemUsed":147141,"TableObjectMemUsed":97000,"TotalBloomFilterMemUsed":448808,"TotalBloomFilterSize":849352,"TotalDiskUsage":717348864,"TotalIndexBlocksSize":4984757,"TotalMemUsed":8494789,"TreeSnapshotMemoryUsed":94336,"WALBufferMemUsed":163840,"WALMemUsed":229456,"WriteAmp":2.2464120388031006,"WriteCacheMemUsed":32760,"WriteCacheQuota":104300248,"keyStats":{"ActiveBloomFilterSize":400960,"ActiveDiskUsage":13934592,"ActiveIndexBlocksSize":156967,"BloomFilterFPR":0.013477307995700231,"BufferMemUsed":0,"BytesIncoming":135528160,"BytesOutgoing":0,"CheckpointOverhead":17539072,"CheckpointOverheadPeriod":183,"CompactReader":{"NBytesOutgoing":0,"NDataBlockReads":55786,"NIndexBlockReads":826,"NReadBytes":113288899,"NReadIOs":31511},"CompactWriter":{"NBytesIncoming":135105262,"NDataBlockWrites":28338,"NIndexBlockWrites":289,"NInserts":2288365,"NSyncs":0,"NWriteBytes":57839616,"NWriteIOs":125},"HistoryDiskUsage":0,"HistoryLogicalDataSize":0,"HistoryLogicalDiskSize":0,"HistorySizeBytesEvicted":0,"HistoryStartTimestamp":0,"HistoryTimeBytesEvicted":0,"IterateReader":{"NBytesOutgoing":0,"NDataBlockReads":0,"NIndexBlockReads":0,"NReadBytes":0,"NReadIOs":0},"IteratorMemUsed":0,"LevelStats":[{"CompactingSize":0,"DataBlocksCompressSize":2361002,"DataBlocksCompressionRatio":1.0,"DataBlocksSize":2361002,"DataBlocksSpaceReductionEstimate":0.0,"HistoryLogicalSize":0,"HistoryPhysicalSize":0,"Level":0,"LogicalSize":2459843,"NWriteBytes":145993728,"NumTables":3,"PhysicalSize":2457600,"SizeRatio":0.08378175326756068,"TargetSize":29360128,"WastedSpace":1362},{"CompactingSize":0,"DataBlocksCompressSize":0,"DataBlocksCompressionRatio":0.0,"DataBlocksSize":0,"DataBlocksSpaceReductionEstimate":0.0,"HistoryLogicalSize":0,"HistoryPhysicalSize":0,"Level":1,"LogicalSize":0,"NWriteBytes":0,"NumTables":0,"PhysicalSize":0,"SizeRatio":0.0,"TargetSize":29360128,"WastedSpace":0},{"CompactingSize":0,"DataBlocksCompressSize":0,"DataBlocksCompressionRatio":0.0,"DataBlocksSize":0,"DataBlocksSpaceReductionEstimate":0.0,"HistoryLogicalSize":0,"HistoryPhysicalSize":0,"Level":2,"LogicalSize":0,"NWriteBytes":0,"NumTables":0,"PhysicalSize":0,"SizeRatio":0.0,"TargetSize":29360128,"WastedSpace":0},{"CompactingSize":0,"DataBlocksCompressSize":10890984,"DataBlocksCompressionRatio":2.1620304464683815,"DataBlocksSize":23546639,"DataBlocksSpaceReductionEstimate":53.74718234734053,"HistoryLogicalSize":0,"HistoryPhysicalSize":0,"Level":3,"LogicalSize":11435900,"NWriteBytes":57839616,"NumTables":7,"PhysicalSize":11476992,"SizeRatio":0.38950443267822266,"TargetSize":29360128,"WastedSpace":41092},{"CompactingSize":0,"DataBlocksCompressSize":0,"DataBlocksCompressionRatio":0.0,"DataBlocksSize":0,"DataBlocksSpaceReductionEstimate":0.0,"HistoryLogicalSize":0,"HistoryPhysicalSize":0,"Level":4,"LogicalSize":0,"NWriteBytes":0,"NumTables":0,"PhysicalSize":0,"SizeRatio":0.0,"TargetSize":29360128,"WastedSpace":0}],"LogicalDataSize":2459843,"LogicalDiskSize":2459843,"ManagedFileObjectMemUsed":2112,"MemtablesMemUsed":10920,"NBloomFilterHits":14895003,"NBloomFilterMisses":203487,"NCompacts":32,"NDataLevelFileCountCompacts":0,"NFailedCompacts":0,"NFileCountCompacts":0,"NInternalFragmentationCompacts":0,"NItemsDel":0,"NItemsDropped":948809,"NItemsGarbageCollected":2188967,"NItemsGet":2898850,"NItemsMerged":0,"NItemsPut":2679925,"NItemsTransformed":2288365,"NNonL0LocalCompacts":0,"NReadBytes":184882081,"NReadIOs":57296,"NRetryCompacts":0,"NSyncs":427,"NTTLCompacts":4,"NTableFiles":22,"NTablesCreated":345,"NTablesDeleted":335,"NWriteBytes":204071351,"NWriteIOs":991,"NWriterCompacts":0,"NonHistoryLogicalDataSize":0,"NonHistoryLogicalDiskSize":0,"NonResidentBloomFilterSize":445446,"PendingCheckpointBytes":0,"ReadAheadBufferMemUsed":0,"ReadAmp":0.0,"ReadIOAmp":0.008894906600893458,"Reader":{"NBytesOutgoing":0,"NDataBlockReads":1499284,"NIndexBlockReads":2877393,"NReadBytes":71593182,"NReadIOs":25785},"TableMetaMemUsed":8766,"TableObjectMemUsed":4400,"TotalBloomFilterMemUsed":448512,"TotalBloomFilterSize":846406,"TotalDiskUsage":31473664,"TotalIndexBlocksSize":331688,"TreeSnapshotMemoryUsed":23792,"UID":"","WriteAmp":1.5057487019671778,"Writer":{"NBytesIncoming":159643885,"NDataBlockWrites":33724,"NIndexBlockWrites":603,"NInserts":2679865,"NSyncs":427,"NWriteBytes":146231735,"NWriteIOs":866}},"localStats":{"ActiveBloomFilterSize":1272,"ActiveDiskUsage":303104,"ActiveIndexBlocksSize":1846,"BloomFilterFPR":0.0,"BufferMemUsed":0,"BytesIncoming":6479143,"BytesOutgoing":0,"CheckpointOverhead":401408,"CheckpointOverheadPeriod":183,"CompactReader":{"NBytesOutgoing":0,"NDataBlockReads":294,"NIndexBlockReads":294,"NReadBytes":188138,"NReadIOs":576},"CompactWriter":{"NBytesIncoming":13171,"NDataBlockWrites":18,"NIndexBlockWrites":18,"NInserts":72,"NSyncs":0,"NWriteBytes":147456,"NWriteIOs":18},"HistoryDiskUsage":0,"HistoryLogicalDataSize":0,"HistoryLogicalDiskSize":0,"HistorySizeBytesEvicted":0,"HistoryStartTimestamp":0,"HistoryTimeBytesEvicted":0,"IterateReader":{"NBytesOutgoing":0,"NDataBlockReads":0,"NIndexBlockReads":0,"NReadBytes":0,"NReadIOs":0},"IteratorMemUsed":0,"LevelStats":[{"CompactingSize":0,"DataBlocksCompressSize":18530,"DataBlocksCompressionRatio":1.0,"DataBlocksSize":18530,"DataBlocksSpaceReductionEstimate":0.0,"HistoryLogicalSize":0,"HistoryPhysicalSize":0,"Level":0,"LogicalSize":274700,"NWriteBytes":2564096,"NumTables":30,"PhysicalSize":245760,"SizeRatio":0.009356226239885603,"TargetSize":29360128,"WastedSpace":0},{"CompactingSize":0,"DataBlocksCompressSize":0,"DataBlocksCompressionRatio":0.0,"DataBlocksSize":0,"DataBlocksSpaceReductionEstimate":0.0,"HistoryLogicalSize":0,"HistoryPhysicalSize":0,"Level":1,"LogicalSize":0,"NWriteBytes":0,"NumTables":0,"PhysicalSize":0,"SizeRatio":0.0,"TargetSize":29360128,"WastedSpace":0},{"CompactingSize":0,"DataBlocksCompressSize":4012,"DataBlocksCompressionRatio":1.339481555333998,"DataBlocksSize":5374,"DataBlocksSpaceReductionEstimate":25.344250093040564,"HistoryLogicalSize":0,"HistoryPhysicalSize":0,"Level":2,"LogicalSize":63960,"NWriteBytes":147456,"NumTables":7,"PhysicalSize":57344,"SizeRatio":0.002178464617047991,"TargetSize":29360128,"WastedSpace":0}],"LogicalDataSize":63960,"LogicalDiskSize":63960,"ManagedFileObjectMemUsed":8256,"MemtablesMemUsed":10920,"NBloomFilterHits":0,"NBloomFilterMisses":0,"NCompacts":18,"NDataLevelFileCountCompacts":0,"NFailedCompacts":0,"NFileCountCompacts":14,"NInternalFragmentationCompacts":0,"NItemsDel":0,"NItemsDropped":0,"NItemsGarbageCollected":23965,"NItemsGet":71,"NItemsMerged":0,"NItemsPut":24053,"NItemsTransformed":0,"NNonL0LocalCompacts":0,"NReadBytes":194030,"NReadIOs":594,"NRetryCompacts":0,"NSyncs":427,"NTTLCompacts":4,"NTableFiles":86,"NTablesCreated":331,"NTablesDeleted":294,"NWriteBytes":2925368,"NWriteIOs":758,"NWriterCompacts":0,"NonHistoryLogicalDataSize":0,"NonHistoryLogicalDiskSize":0,"NonResidentBloomFilterSize":1674,"PendingCheckpointBytes":0,"ReadAheadBufferMemUsed":0,"ReadAmp":0.0,"ReadIOAmp":0.2535211267605634,"Reader":{"NBytesOutgoing":0,"NDataBlockReads":9,"NIndexBlockReads":9,"NReadBytes":5892,"NReadIOs":18},"TableMetaMemUsed":26029,"TableObjectMemUsed":17200,"TotalBloomFilterMemUsed":296,"TotalBloomFilterSize":2946,"TotalDiskUsage":704512,"TotalIndexBlocksSize":4238,"TreeSnapshotMemoryUsed":24608,"UID":"","WriteAmp":0.45150539199397205,"Writer":{"NBytesIncoming":178050,"NDataBlockWrites":313,"NIndexBlockWrites":313,"NInserts":640,"NSyncs":427,"NWriteBytes":2777912,"NWriteIOs":740}},"seqStats":{"ActiveBloomFilterSize":0,"ActiveDiskUsage":509861888,"ActiveIndexBlocksSize":3470115,"BloomFilterFPR":0.0,"BufferMemUsed":3486016,"BytesIncoming":4812974583,"BytesOutgoing":0,"CheckpointOverhead":72908800,"CheckpointOverheadPeriod":183,"CompactReader":{"NBytesOutgoing":0,"NDataBlockReads":1657690,"NIndexBlockReads":16307,"NReadBytes":5067435259,"NReadIOs":1673997},"CompactWriter":{"NBytesIncoming":2743609774,"NDataBlockWrites":660213,"NIndexBlockWrites":9000,"NInserts":5289661,"NSyncs":0,"NWriteBytes":2038829056,"NWriteIOs":7930},"HistoryDiskUsage":36577280,"HistoryLogicalDataSize":35159034,"HistoryLogicalDiskSize":52798312,"HistorySizeBytesEvicted":2581094400,"HistoryStartTimestamp":1683055156,"HistoryTimeBytesEvicted":867528704,"IterateReader":{"NBytesOutgoing":0,"NDataBlockReads":0,"NIndexBlockReads":0,"NReadBytes":0,"NReadIOs":0},"IteratorMemUsed":11552,"LevelStats":[{"CompactingSize":0,"DataBlocksCompressSize":1289488,"DataBlocksCompressionRatio":1.0,"DataBlocksSize":1289488,"DataBlocksSpaceReductionEstimate":0.0,"HistoryLogicalSize":0,"HistoryPhysicalSize":0,"Level":0,"LogicalSize":10538775,"NWriteBytes":50667520,"NumTables":3,"PhysicalSize":1323008,"SizeRatio":0.3589485372815813,"TargetSize":29360128,"WastedSpace":1425},{"CompactingSize":0,"DataBlocksCompressSize":0,"DataBlocksCompressionRatio":0.0,"DataBlocksSize":0,"DataBlocksSpaceReductionEstimate":0.0,"HistoryLogicalSize":0,"HistoryPhysicalSize":0,"Level":1,"LogicalSize":0,"NWriteBytes":0,"NumTables":0,"PhysicalSize":0,"SizeRatio":0.0,"TargetSize":29360128,"WastedSpace":0},{"CompactingSize":4195518,"DataBlocksCompressSize":480711,"DataBlocksCompressionRatio":2.4730388944708985,"DataBlocksSize":1188817,"DataBlocksSpaceReductionEstimate":59.563919425782096,"HistoryLogicalSize":0,"HistoryPhysicalSize":0,"Level":2,"LogicalSize":52126972,"NWriteBytes":49418240,"NumTables":26,"PhysicalSize":692224,"SizeRatio":1.7754340853009904,"TargetSize":29360128,"WastedSpace":294},{"CompactingSize":60827297,"DataBlocksCompressSize":1346410,"DataBlocksCompressionRatio":2.0674304260960628,"DataBlocksSize":2783609,"DataBlocksSpaceReductionEstimate":51.63077860432266,"HistoryLogicalSize":17639278,"HistoryPhysicalSize":0,"Level":3,"LogicalSize":146749382,"NWriteBytes":62824448,"NumTables":71,"PhysicalSize":2072576,"SizeRatio":1.1224110932203957,"TargetSize":115029248,"WastedSpace":31},{"CompactingSize":115245751,"DataBlocksCompressSize":499959487,"DataBlocksCompressionRatio":1.4078367913838588,"DataBlocksSize":703861360,"DataBlocksSpaceReductionEstimate":28.96903915850701,"HistoryLogicalSize":52798312,"HistoryPhysicalSize":36577280,"Level":4,"LogicalSize":575146258,"NWriteBytes":5483675648,"NumTables":78,"PhysicalSize":505774080,"SizeRatio":0.6065060047558478,"TargetSize":861241178,"WastedSpace":1399065}],"LogicalDataSize":482815330,"LogicalDiskSize":690392009,"ManagedFileObjectMemUsed":36192,"MemtablesMemUsed":10920,"NBloomFilterHits":0,"NBloomFilterMisses":0,"NCompacts":1750,"NDataLevelFileCountCompacts":0,"NFailedCompacts":0,"NFileCountCompacts":0,"NInternalFragmentationCompacts":4,"NItemsDel":0,"NItemsDropped":0,"NItemsGarbageCollected":1973757,"NItemsGet":0,"NItemsMerged":1973757,"NItemsPut":3975785,"NItemsTransformed":0,"NNonL0LocalCompacts":0,"NReadBytes":5067435259,"NReadIOs":1673997,"NRetryCompacts":1291,"NSyncs":427,"NTTLCompacts":0,"NTableFiles":385,"NTablesCreated":4858,"NTablesDeleted":4521,"NWriteBytes":5609838845,"NWriteIOs":15539,"NWriterCompacts":0,"NonHistoryLogicalDataSize":0,"NonHistoryLogicalDiskSize":0,"NonResidentBloomFilterSize":0,"PendingCheckpointBytes":121606144,"ReadAheadBufferMemUsed":0,"ReadAmp":0.0,"ReadIOAmp":0.0,"Reader":{"NBytesOutgoing":0,"NDataBlockReads":0,"NIndexBlockReads":0,"NReadBytes":0,"NReadIOs":0},"TableMetaMemUsed":112346,"TableObjectMemUsed":75400,"TotalBloomFilterMemUsed":0,"TotalBloomFilterSize":0,"TotalDiskUsage":685170688,"TotalIndexBlocksSize":4648831,"TreeSnapshotMemoryUsed":45936,"UID":"","WriteAmp":1.1655658570927467,"Writer":{"NBytesIncoming":4868917140,"NDataBlockWrites":1166826,"NIndexBlockWrites":9006,"NInserts":4892171,"NSyncs":427,"NWriteBytes":3571009789,"NWriteIOs":7609}},"walStats":{"DiskUsed":5992448,"NActiveSegments":1,"NInserts":2740039,"NLoggedBytes":4899557376,"NSegments":164,"NSyncs":12022,"NTxns":12022,"NWriteBytes":4899557376,"NWrites":17866,"PreallocateSupported":true,"TxnSizeEstimate":53248}}

 rw_6:numLoadedVb:             0

 rw_6:open:                    0

QE-TEST:

guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/testexec.52316.ini bucket_storage=magma,GROUP=P0,bucket_eviction_policy=fullEviction,randomize_value=True,rerun=false,replicas=1,test_itr=2,get-cbcollect-info=True,durability=MAJORITY,infra_log_level=debug,log_level=debug,enable_dp=false,get-cbcollect-info=True,autoCompactionDefined=true,bucket_history_retention_seconds=600,bucket_history_retention_bytes=6000000000,upgrade_version=7.2.0-5321 -t storage.magma.magma_upserts.BasicUpsertTests.test_multi_update_delete,deep_copy=True,upgrade_version=7.2.0-5321,fragmentation=20,rerun=false,test_itr=2,update_itr=2,GROUP=P0,enable_dp=false,bucket_history_retention_seconds=600,sdk_timeout=60,bucket_history_retention_bytes=6000000000,get-cbcollect-info=True,replicas=1,durability=MAJORITY,bucket_storage=magma,log_level=debug,vbuckets=128,bucket_eviction_policy=fullEviction,nodes_init=4,num_items=7000000,autoCompactionDefined=true,randomize_value=True,infra_log_level=debug'