Details
-
Bug
-
Resolution: Fixed
-
Critical
-
6.5.0
-
Triaged
-
-
Unknown
Description
Build : 6.5.0-4659
While testing the scenario mentioned in MB-36613, from the GSI stats, the memory used and disk used for the index is way too high compared to the data size.
HQ-Computer:cbcollect_info_ns_1@172.23.104.21_20191023-162930 mihirkamdar$ cat stats.txt | jq
|
{
|
"MAINT_STREAM/172.23.104.16:8091/projector_latency": 15244749,
|
"MAINT_STREAM/172.23.104.17:8091/projector_latency": 14651174,
|
"MAINT_STREAM/172.23.104.19:8091/projector_latency": 12170982,
|
"cpu_utilization": 551.4333333333333,
|
"index_not_found_errcount": 0,
|
"indexer_state": "Active",
|
"memory_free": 203866112,
|
"memory_quota": 21474836480,
|
"memory_rss": 22438375424,
|
"memory_total": 25107591168,
|
"memory_total_storage": 21538320384,
|
"memory_used": 22156171264,
|
"memory_used_queue": 0,
|
"memory_used_storage": 18740405192,
|
"needs_restart": false,
|
"num_connections": 1,
|
"num_cpu_core": 8,
|
"storage_mode": "plasma",
|
"test:idx1:arrkey_size_distribution": {
|
"(0-64)": 0,
|
"(102401-max)": 9999,
|
"(1025-4096)": 0,
|
"(257-1024)": 0,
|
"(4097-102400)": 0,
|
"(65-256)": 40051
|
},
|
"test:idx1:avg_array_length": 2,
|
"test:idx1:avg_disk_bps": 12266738,
|
"test:idx1:avg_drain_rate": 8,
|
"test:idx1:avg_mutation_rate": 1,
|
"test:idx1:avg_scan_latency": 0,
|
"test:idx1:avg_scan_rate": 0,
|
"test:idx1:avg_scan_request_latency": 0,
|
"test:idx1:avg_scan_wait_latency": 0,
|
"test:idx1:avg_ts_interval": 53713312,
|
"test:idx1:avg_ts_items_count": 1,
|
"test:idx1:backstore_data_size": 1029532130,
|
"test:idx1:build_progress": 100,
|
"test:idx1:cache_hit_percent": 77,
|
"test:idx1:cache_hits": 2265900,
|
"test:idx1:cache_misses": 649409,
|
"test:idx1:client_cancel_errcount": 0,
|
"test:idx1:data_size": 2060442259,
|
"test:idx1:delete_bytes": 0,
|
"test:idx1:disk_load_duration": 0,
|
"test:idx1:disk_size": 90639855850,
|
"test:idx1:disk_store_duration": 192445,
|
"test:idx1:docid_count": 50050,
|
"test:idx1:flush_queue_size": 1,
|
"test:idx1:frag_percent": 0,
|
"test:idx1:get_bytes": 2286073184256,
|
"test:idx1:insert_bytes": 1294446034944,
|
"test:idx1:items_count": 100100,
|
"test:idx1:key_size_distribution": {
|
"(0-64)": 0,
|
"(102401-max)": 0,
|
"(1025-4096)": 0,
|
"(257-1024)": 0,
|
"(4097-102400)": 19997,
|
"(65-256)": 80102
|
},
|
"test:idx1:key_size_stats_since": 1571806079429022700,
|
"test:idx1:last_known_scan_time": 0,
|
"test:idx1:last_rollback_time": "1571806078844572665",
|
"test:idx1:memory_used": 18742253783,
|
"test:idx1:not_ready_errcount": 0,
|
"test:idx1:num_commits": 136,
|
"test:idx1:num_compactions": 0,
|
"test:idx1:num_completed_requests": 0,
|
"test:idx1:num_docs_indexed": 753852,
|
"test:idx1:num_docs_pending": 0,
|
"test:idx1:num_docs_processed": 753852,
|
"test:idx1:num_docs_queued": 4224,
|
"test:idx1:num_flush_queued": 753853,
|
"test:idx1:num_items_flushed": 2915308,
|
"test:idx1:num_items_restored": 0,
|
"test:idx1:num_last_snapshot_reply": 0,
|
"test:idx1:num_open_snapshots": 1,
|
"test:idx1:num_requests": 0,
|
"test:idx1:num_rows_returned": 0,
|
"test:idx1:num_rows_scanned": 0,
|
"test:idx1:num_scan_errors": 0,
|
"test:idx1:num_scan_timeouts": 0,
|
"test:idx1:num_snapshot_waiters": 0,
|
"test:idx1:num_snapshots": 703237,
|
"test:idx1:progress_stat_time": "1571847015538386637",
|
"test:idx1:recs_in_mem": 29434,
|
"test:idx1:recs_on_disk": 406854,
|
"test:idx1:resident_percent": 6,
|
"test:idx1:scan_bytes_read": 0,
|
"test:idx1:scan_wait_duration": 0,
|
"test:idx1:since_last_snapshot": 55272296,
|
"test:idx1:total_scan_duration": 0,
|
"test:idx1:total_scan_request_duration": 0,
|
"test:mutation_queue_size": 4320,
|
"test:num_mutations_queued": 758173,
|
"test:num_nonalign_ts": 0,
|
"test:num_rollbacks": 0,
|
"test:timings/dcp_getseqs": "6830 10275955872 20590604552770336",
|
"test:ts_queue_size": 4296,
|
"timestamp": "1571847021213681396",
|
"timings/stats_response": "49310 64981647440 199514689431401022",
|
"uptime": "11h25m0.885106566s"
|
}
|
"test:idx1:data_size": 2060442259 (1.91GB)
"test:idx1:disk_size": 90639855850 (84.41GB)
"memory_used": 22156171264 (20.63 GB)
"memory_used_storage": 18740405192 (17.45GB)
So, the disk+memory used by the index with data_size of 2GB is > 100GB, which doesn't look right. The workload ran for 11 hrs and the resource utilization did not come down.
Storage stats around this time:
==== StorageStats ====
|
[
|
{
|
"Index": "test:idx1", "Id": 730860754992284375, "PartitionId": 0,
|
"Stats":
|
{
|
"MainStore":
|
{
|
"memory_quota": 18940805773,
|
"punch_hole_support": true,
|
"count": 2911904,
|
"compacts": 1216557,
|
"purges": 0,
|
"splits": 275239,
|
"merges": 106292,
|
"inserts": 2911904,
|
"deletes": 0,
|
"compact_conflicts": 153,
|
"split_conflicts": 204,
|
"merge_conflicts": 0,
|
"insert_conflicts": 866,
|
"delete_conflicts": 0,
|
"swapin_conflicts": 0,
|
"persist_conflicts": 13,
|
"memory_size": 10364866142,
|
"memory_size_index": 8418770331,
|
"allocated": 883064758763,
|
"freed": 872699892621,
|
"reclaimed": 872697328498,
|
"reclaim_pending": 2564123,
|
"reclaim_list_size": 91386387,
|
"reclaim_list_count": 620,
|
"reclaim_threshold": 2,
|
"allocated_index": 13565357631,
|
"freed_index": 5146587300,
|
"reclaimed_index": 5146536017,
|
"num_pages": 168948,
|
"items_count": 100100,
|
"total_records": 435512,
|
"num_rec_allocs": 13261691,
|
"num_rec_frees": 13229986,
|
"num_rec_swapout": 7310177,
|
"num_rec_swapin": 6906370,
|
"bytes_incoming": 143071639532,
|
"bytes_written": 689071841280,
|
"write_amp": 5.37,
|
"write_amp_avg": 4.82,
|
"lss_gc_status": "frag 30, data: 47180392952, used: 68376806167, relocated: 1181884, retries: 93514, skipped: 1013998 log:(620693081321 - 689069887488)",
|
"lss_fragmentation": 31,
|
"lss_data_size": 47180598444,
|
"lss_recoverypt_size": 163840,
|
"lss_maxsn_size": 4096,
|
"lss_used_space": 68381958144,
|
"checkpoint_used_space":16836948085,
|
"lss_num_reads": 4533280,
|
"lss_read_bs": 1129048693619,
|
"lss_blk_read_bs": 1128525692928,
|
"lss_gc_num_reads": 3818866,
|
"lss_gc_reads_bs": 934003504526,
|
"lss_blk_gc_reads_bs": 951519924224,
|
"lss_rdr_reads_bs": 0,
|
"lss_blk_rdr_reads_bs": 0,
|
"cache_hits": 2263810,
|
"cache_misses": 648094,
|
"cache_hit_ratio": 0.60286,
|
"rlss_num_reads": 0,
|
"rcache_hits": 0,
|
"rcache_misses": 0,
|
"rcache_hit_ratio": 0.00000,
|
"resident_ratio": 0.07280,
|
"mvcc_purge_ratio": 4.35077,
|
"currSn": 702388,
|
"gcSn": 702386,
|
"gcSnIntervals": "[0 679274 684430 689581 694763 699961 702387]",
|
"purger_running": false,
|
"mem_throttled": false,
|
"lss_throttled": false,
|
"lss_head_offset": 620693081321,
|
"lss_tail_offset": 689071841280,
|
"num_wctxs": 22,
|
"num_readers": 24,
|
"num_writers": 8,
|
"buf_memused": 94027541,
|
"page_bytes": -1768164427,
|
"page_cnt": 9883,
|
"page_itemcnt": -32575,
|
"avg_item_size": 0,
|
"avg_page_size": -178909,
|
"act_max_page_items": 4,
|
"act_min_page_items": 1,
|
"act_max_delta_len": 100,
|
"est_resident_mem": 145059539213
|
},
|
"BackStore":
|
{
|
"memory_quota": 18940805773,
|
"punch_hole_support": true,
|
"count": 1455952,
|
"compacts": 397990,
|
"purges": 0,
|
"splits": 10994,
|
"merges": 326,
|
"inserts": 1455952,
|
"deletes": 0,
|
"compact_conflicts": 218,
|
"split_conflicts": 125,
|
"merge_conflicts": 0,
|
"insert_conflicts": 467,
|
"delete_conflicts": 0,
|
"swapin_conflicts": 0,
|
"persist_conflicts": 114,
|
"memory_size": 156641206,
|
"memory_size_index": 665123,
|
"allocated": 873821934500,
|
"freed": 873665293294,
|
"reclaimed": 873665293123,
|
"reclaim_pending": 171,
|
"reclaim_list_size": 4100850,
|
"reclaim_list_count": 17,
|
"reclaim_threshold": 1,
|
"allocated_index": 685287,
|
"freed_index": 20164,
|
"reclaimed_index": 20164,
|
"num_pages": 10669,
|
"items_count": 50050,
|
"total_records": 98917,
|
"num_rec_allocs": 20639355,
|
"num_rec_frees": 20620579,
|
"num_rec_swapout": 13803812,
|
"num_rec_swapin": 13723671,
|
"bytes_incoming": 72036012396,
|
"bytes_written": 604006903808,
|
"write_amp": 0.67,
|
"write_amp_avg": 8.38,
|
"lss_gc_status": "frag 30, data: 3463294468, used: 5014716165, relocated: 1311104, retries: 174, skipped: 457203 log:(598983394006 - 603998134272)",
|
"lss_fragmentation": 30,
|
"lss_data_size": 3472333293,
|
"lss_recoverypt_size": 163840,
|
"lss_maxsn_size": 4096,
|
"lss_used_space": 5060292608,
|
"checkpoint_used_space":117,
|
"lss_num_reads": 4086670,
|
"lss_read_bs": 1385731435261,
|
"lss_blk_read_bs": 1155309928448,
|
"lss_gc_num_reads": 2931697,
|
"lss_gc_reads_bs": 965876711861,
|
"lss_blk_gc_reads_bs": 976206221312,
|
"lss_rdr_reads_bs": 402968823967,
|
"lss_blk_rdr_reads_bs": 178933850112,
|
"cache_hits": 1579073,
|
"cache_misses": 579830,
|
"cache_hit_ratio": 0.67939,
|
"rlss_num_reads": 1076305,
|
"rcache_hits": 201652,
|
"rcache_misses": 501299,
|
"rcache_hit_ratio": 0.04545,
|
"resident_ratio": 0.18982,
|
"mvcc_purge_ratio": 1.97636,
|
"currSn": 702388,
|
"gcSn": 702386,
|
"gcSnIntervals": "[0 679274 684430 689581 694763 699961 702387]",
|
"purger_running": false,
|
"mem_throttled": false,
|
"lss_throttled": false,
|
"lss_head_offset": 598983394006,
|
"lss_tail_offset": 604006903808,
|
"num_wctxs": 22,
|
"num_readers": 0,
|
"num_writers": 8,
|
"buf_memused": 91265010,
|
"page_bytes": 207389216,
|
"page_cnt": 553,
|
"page_itemcnt": 20568,
|
"avg_item_size": 10083,
|
"avg_page_size": 375025,
|
"act_max_page_items": 19,
|
"act_min_page_items": 1,
|
"act_max_delta_len": 30,
|
"est_resident_mem": 5929985332
|
}}
|
}
|
]
|