Details
-
Bug
-
Resolution: Fixed
-
Critical
-
6.5.0
-
Triaged
-
-
Unknown
Description
Build : 6.5.0-4659
Cluster info
- 5 nodes = 3 kv + 1 query + 1 gsi
- 24 GB RAM 8 core each VM
- 20480 MB RAM quota for indexer node
Steps
1. Create a bucket called test and and index idx1 on it
create index idx1 on test(id, name, DISTINCT ARRAY v.element2 FOR v IN `array_field` END)
2. Load 50K docs with Initial key size = 100 bytes (2 array items with 50 bytes each)
Sample doc :
{
|
"array_field": [
|
{
|
"element1": "Lg",
|
"element2": "idqrKkGRGygTVwrlEGxZuqsNxrICySsooUxDBLqzfEDDydMzn"
|
},
|
{
|
"element1": "yJ",
|
"element2": "GTzWOWBdGKyWLIMfqJIiiFopRkKhsswxHkdTDxDIMkDOVtjvV"
|
}
|
],
|
"id": "45000",
|
"name": "ZMfLeedOl"
|
}
|
3. Change the following settings & restart indexer
curl http://172.23.104.21:9102/settings -u <username:password> -X POST -d "
"
Checkpoint interval is 1 -> curl http://172.23.104.21:9102/settings -u <username:password>-X POST -d "
"
persistent snapshot interval is 5m ->
curl http://172.23.104.21:9102/settings -u <username:password> -X POST -d "
"
curl http://172.23.104.21:9102/settings -u <username:password> -X POST -d "
"
4. Create & Build initial index
5. Once Step 4 is done, run Random scans on a range of 10K documents (10K docs)
6. Along with Step 5, run update workload on another set of 10K documents such that the doc size for these docs is 100K (50K bytes per array item). This range of docs should be separate from the one in #5 above.
Check the frag_percent stats for gsi, and lss_fragmentation plasma stat while Steps 5 & 6 are running. They do not match. For eg.
2019-10-23T09:05:21.209-07:00 [Info] PeriodicStats = {"MAINT_STREAM/172.23.104.16:8091/projector_latency":14384947,"MAINT_STREAM/172.23.104.17:8091/projector_latency":13133763,"MAINT_STREAM/172.23.104.19:8091/projector_latency":13855137,"cpu_utilization":236.10000000000002,"index_not_found_errcount":0,"indexer_state":"Active","memory_free":205983744,"memory_quota":21474836480,"memory_rss":21327147008,"memory_total":25107591168,"memory_total_storage":21526118400,"memory_used":21638612992,"memory_used_queue":0,"memory_used_storage":18941184715,"needs_restart":false,"num_connections":1,"num_cpu_core":8,"storage_mode":"plasma","test:idx1:arrkey_size_distribution":{"(0-64)":0,"(102401-max)":9999,"(1025-4096)":0,"(257-1024)":0,"(4097-102400)":0,"(65-256)":40051},"test:idx1:avg_array_length":2,"test:idx1:avg_disk_bps":127436161,"test:idx1:avg_drain_rate":68,"test:idx1:avg_mutation_rate":16,"test:idx1:avg_scan_latency":0,"test:idx1:avg_scan_rate":0,"test:idx1:avg_scan_request_latency":0,"test:idx1:avg_scan_wait_latency":0,"test:idx1:avg_ts_interval":54116970,"test:idx1:avg_ts_items_count":1,"test:idx1:backstore_data_size":1029532130,"test:idx1:build_progress":100,"test:idx1:cache_hit_percent":77,"test:idx1:cache_hits":2263614,"test:idx1:cache_misses":647978,"test:idx1:client_cancel_errcount":0,"test:idx1:data_size":2060493490,"test:idx1:delete_bytes":0,"test:idx1:disk_load_duration":0,"test:idx1:disk_size":90294317290,"test:idx1:disk_store_duration":192445,"test:idx1:docid_count":50050,"test:idx1:flush_queue_size":1,"test:idx1:frag_percent":0,"test:idx1:get_bytes":2283633197056,"test:idx1:insert_bytes":1292959645696,"test:idx1:items_count":100100,"test:idx1:key_size_distribution":{"(0-64)":0,"(102401-max)":0,"(1025-4096)":0,"(257-1024)":0,"(4097-102400)":19998,"(65-256)":80102},"test:idx1:key_size_stats_since":1571806079429022791,"test:idx1:last_known_scan_time":0,"test:idx1:last_rollback_time":"1571806078844572665","test:idx1:memory_used":18944133750,"test:idx1:not_ready_errcount":0,"test:idx1:num_commits":136,"test:idx1:num_compactions":0,"test:idx1:num_completed_requests":0,"test:idx1:num_docs_indexed":753001,"test:idx1:num_docs_pending":0,"test:idx1:num_docs_processed":752901,"test:idx1:num_docs_queued":1,"test:idx1:num_flush_queued":753002,"test:idx1:num_items_flushed":2911904,"test:idx1:num_items_restored":0,"test:idx1:num_last_snapshot_reply":0,"test:idx1:num_open_snapshots":1,"test:idx1:num_requests":0,"test:idx1:num_rows_returned":0,"test:idx1:num_rows_scanned":0,"test:idx1:num_scan_errors":0,"test:idx1:num_scan_timeouts":0,"test:idx1:num_snapshot_waiters":0,"test:idx1:num_snapshots":702387,"test:idx1:progress_stat_time":"1571846715528298013","test:idx1:recs_in_mem":31084,"test:idx1:recs_on_disk":404374,"test:idx1:resident_percent":7,"test:idx1:scan_bytes_read":0,"test:idx1:scan_wait_duration":0,"test:idx1:since_last_snapshot":64321930,"test:idx1:total_scan_duration":0,"test:idx1:total_scan_request_duration":0,"test:mutation_queue_size":0,"test:num_mutations_queued":753002,"test:num_nonalign_ts":0,"test:num_rollbacks":0,"test:timings/dcp_getseqs":"6780 9994595542 18274443096207344","test:ts_queue_size":0,"timestamp":"1571846721209161211","timings/stats_response":"48950 64518278659 188737222218129853","uptime":"11h20m0.880585644s"}
|
==== StorageStats ====
|
[
|
{
|
"Index": "test:idx1", "Id": 730860754992284375, "PartitionId": 0,
|
"Stats":
|
{
|
"MainStore":
|
{
|
"memory_quota": 18940805773,
|
"punch_hole_support": true,
|
"count": 2911904,
|
"compacts": 1216557,
|
"purges": 0,
|
"splits": 275239,
|
"merges": 106292,
|
"inserts": 2911904,
|
"deletes": 0,
|
"compact_conflicts": 153,
|
"split_conflicts": 204,
|
"merge_conflicts": 0,
|
"insert_conflicts": 866,
|
"delete_conflicts": 0,
|
"swapin_conflicts": 0,
|
"persist_conflicts": 13,
|
"memory_size": 10364866142,
|
"memory_size_index": 8418770331,
|
"allocated": 883064758763,
|
"freed": 872699892621,
|
"reclaimed": 872697328498,
|
"reclaim_pending": 2564123,
|
"reclaim_list_size": 91386387,
|
"reclaim_list_count": 620,
|
"reclaim_threshold": 2,
|
"allocated_index": 13565357631,
|
"freed_index": 5146587300,
|
"reclaimed_index": 5146536017,
|
"num_pages": 168948,
|
"items_count": 100100,
|
"total_records": 435512,
|
"num_rec_allocs": 13261691,
|
"num_rec_frees": 13229986,
|
"num_rec_swapout": 7310177,
|
"num_rec_swapin": 6906370,
|
"bytes_incoming": 143071639532,
|
"bytes_written": 689071841280,
|
"write_amp": 5.37,
|
"write_amp_avg": 4.82,
|
"lss_gc_status": "frag 30, data: 47180392952, used: 68376806167, relocated: 1181884, retries: 93514, skipped: 1013998 log:(620693081321 - 689069887488)",
|
"lss_fragmentation": 31,
|
"lss_data_size": 47180598444,
|
"lss_recoverypt_size": 163840,
|
"lss_maxsn_size": 4096,
|
"lss_used_space": 68381958144,
|
"checkpoint_used_space":16836948085,
|
"lss_num_reads": 4533280,
|
"lss_read_bs": 1129048693619,
|
"lss_blk_read_bs": 1128525692928,
|
"lss_gc_num_reads": 3818866,
|
"lss_gc_reads_bs": 934003504526,
|
"lss_blk_gc_reads_bs": 951519924224,
|
"lss_rdr_reads_bs": 0,
|
"lss_blk_rdr_reads_bs": 0,
|
"cache_hits": 2263810,
|
"cache_misses": 648094,
|
"cache_hit_ratio": 0.60286,
|
"rlss_num_reads": 0,
|
"rcache_hits": 0,
|
"rcache_misses": 0,
|
"rcache_hit_ratio": 0.00000,
|
"resident_ratio": 0.07280,
|
"mvcc_purge_ratio": 4.35077,
|
"currSn": 702388,
|
"gcSn": 702386,
|
"gcSnIntervals": "[0 679274 684430 689581 694763 699961 702387]",
|
"purger_running": false,
|
"mem_throttled": false,
|
"lss_throttled": false,
|
"lss_head_offset": 620693081321,
|
"lss_tail_offset": 689071841280,
|
"num_wctxs": 22,
|
"num_readers": 24,
|
"num_writers": 8,
|
"buf_memused": 94027541,
|
"page_bytes": -1768164427,
|
"page_cnt": 9883,
|
"page_itemcnt": -32575,
|
"avg_item_size": 0,
|
"avg_page_size": -178909,
|
"act_max_page_items": 4,
|
"act_min_page_items": 1,
|
"act_max_delta_len": 100,
|
"est_resident_mem": 145059539213
|
},
|
"BackStore":
|
{
|
"memory_quota": 18940805773,
|
"punch_hole_support": true,
|
"count": 1455952,
|
"compacts": 397990,
|
"purges": 0,
|
"splits": 10994,
|
"merges": 326,
|
"inserts": 1455952,
|
"deletes": 0,
|
"compact_conflicts": 218,
|
"split_conflicts": 125,
|
"merge_conflicts": 0,
|
"insert_conflicts": 467,
|
"delete_conflicts": 0,
|
"swapin_conflicts": 0,
|
"persist_conflicts": 114,
|
"memory_size": 156641206,
|
"memory_size_index": 665123,
|
"allocated": 873821934500,
|
"freed": 873665293294,
|
"reclaimed": 873665293123,
|
"reclaim_pending": 171,
|
"reclaim_list_size": 4100850,
|
"reclaim_list_count": 17,
|
"reclaim_threshold": 1,
|
"allocated_index": 685287,
|
"freed_index": 20164,
|
"reclaimed_index": 20164,
|
"num_pages": 10669,
|
"items_count": 50050,
|
"total_records": 98917,
|
"num_rec_allocs": 20639355,
|
"num_rec_frees": 20620579,
|
"num_rec_swapout": 13803812,
|
"num_rec_swapin": 13723671,
|
"bytes_incoming": 72036012396,
|
"bytes_written": 604006903808,
|
"write_amp": 0.67,
|
"write_amp_avg": 8.38,
|
"lss_gc_status": "frag 30, data: 3463294468, used: 5014716165, relocated: 1311104, retries: 174, skipped: 457203 log:(598983394006 - 603998134272)",
|
"lss_fragmentation": 30,
|
"lss_data_size": 3472333293,
|
"lss_recoverypt_size": 163840,
|
"lss_maxsn_size": 4096,
|
"lss_used_space": 5060292608,
|
"checkpoint_used_space":117,
|
"lss_num_reads": 4086670,
|
"lss_read_bs": 1385731435261,
|
"lss_blk_read_bs": 1155309928448,
|
"lss_gc_num_reads": 2931697,
|
"lss_gc_reads_bs": 965876711861,
|
"lss_blk_gc_reads_bs": 976206221312,
|
"lss_rdr_reads_bs": 402968823967,
|
"lss_blk_rdr_reads_bs": 178933850112,
|
"cache_hits": 1579073,
|
"cache_misses": 579830,
|
"cache_hit_ratio": 0.67939,
|
"rlss_num_reads": 1076305,
|
"rcache_hits": 201652,
|
"rcache_misses": 501299,
|
"rcache_hit_ratio": 0.04545,
|
"resident_ratio": 0.18982,
|
"mvcc_purge_ratio": 1.97636,
|
"currSn": 702388,
|
"gcSn": 702386,
|
"gcSnIntervals": "[0 679274 684430 689581 694763 699961 702387]",
|
"purger_running": false,
|
"mem_throttled": false,
|
"lss_throttled": false,
|
"lss_head_offset": 598983394006,
|
"lss_tail_offset": 604006903808,
|
"num_wctxs": 22,
|
"num_readers": 0,
|
"num_writers": 8,
|
"buf_memused": 91265010,
|
"page_bytes": 207389216,
|
"page_cnt": 553,
|
"page_itemcnt": 20568,
|
"avg_item_size": 10083,
|
"avg_page_size": 375025,
|
"act_max_page_items": 19,
|
"act_min_page_items": 1,
|
"act_max_delta_len": 30,
|
"est_resident_mem": 5929985332
|
}}
|
}
|
]
|
