Uploaded image for project: 'Couchbase Server'
  1. Couchbase Server
  2. MB-57763

Investigate avg_page_size < avg_item_size

    XMLWordPrintable

Details

    • Untriaged
    • 0
    • Unknown

    Description

      During a CBSE investigation, on one of the unaffected nodes (node1), I come across stats for an instance where there were periods during which avg_page_size < avg_item_size. Below are the two successive time intervals. There were no change in inserts, items_count and total_records, no splits, no merges. (There was increase in compacts count, but that does not necessarily mean page was compacted).  

      Instance :indexstorage_iWatchXCustomerJourney:Idx_caseId_hitDisp_IWXCJICR_v2:11515755198712134866:1  

      2023-06-11T22:35:18.747+00:00 

                   "act_max_delta_len": 2,
                   "act_max_page_items": 4,
                   "act_min_page_items": 1,
                   "allocated": 23229781419,
                   "allocated_index": 539036705,
                   "avg_item_size": 3068,
                   "avg_page_size": 4522,

                   "page_bytes": 5933678,
                   "page_bytes_compressed": 1450977430,
                   "page_bytes_marshalled": 13390062693,
                   "page_cnt": 1312,
                   "page_itemcnt": 1934,

      2023-06-11T22:52:19.086+00:00

                  "act_max_delta_len": 2,
                  "act_max_page_items": 4,
                  "act_min_page_items": 1,
                  "avg_item_size": 10057,
                  "avg_page_size": 4971,             

                  "page_bytes": 4747328,
                  "page_bytes_compressed": 1452011158,
                  "page_bytes_marshalled": 13399840267,
                  "page_cnt": 955,
                  "page_itemcnt": 472,

      I looked at computePageStats2 and addPageStats which seemed ok. With respect to work context trimming, we are merging the related stats which seemed ok. The stats are then sampled in (s *Plasma) computePageStats(). Need to investigate this more.

      Detailed Stats :

      2023-06-11T22:35:18.747+00:00 

      indexstorage_iWatchXCustomerJourney:Idx_caseId_hitDisp_IWXCJICR_v2:11515755198712134866:1 

      {"slice_0":{"BackStore":{"act_max_delta_len":30,"act_max_page_items":300,"    act_min_page_items":5,"allocated":230250257,"allocated_index":4287,"avg_item_size":125,"avg_page_size":27079,"bloom_false_positives":0,"bloom_fp_rate":0,"bloom_negatives":0,"bloom_tests":0,"byte    s_incoming":22557841,"cache_hit_ratio":0,"cache_hits":1096308,"cache_misses":36209,"checkpoint_used_space":0,"compact_conflicts":0,"compacts":5747,"compacts_fullmarshal":5176,"compress_cache_hit    _ratio":0,"compress_cache_hits":43789,"compress_cache_misses":1052519,"compressed_compressible_percent":0,"compressed_percent":0,"compresses":440,"compression_ratio":1.73251,"count":17835,"currS    n":25287,"decompresses":44208,"delete_conflicts":0,"deletes":0,"est_disk_size":28014200,"est_recovery_size":16460640,"est_resident_mem":2525565,"freed":230245162,"freed_index":0,"gcSn":25285,"gc    SnIntervals":"[0 25285 25286]","insert_conflicts":0,"inserts":17835,"items_count":8917,"lss_blk_rdr_reads_bs":42287104,"lss_blk_read_bs":51412992,"lss_data_size":840426,"lss_maxsn_size":4096,"ls    s_num_reads":36741,"lss_rdr_reads_bs":447211910,"lss_read_bs":456162126,"lss_recoverypt_size":77824,"memory_quota":1073741824,"memory_size":5095,"memory_size_bloom_filter":0,"memory_size_delta":    2816,"memory_size_index":4287,"merge_conflicts":0,"merges":0,"merges_emptypages":0,"mvcc_purge_ratio":1,"num_active_smr":0,"num_burst_visits":94244,"num_compressed_pages":0,"num_free_wctxs":3,"n    um_pages":44,"num_pending_smr":0,"num_periodic_visits":133574281,"num_readers":0,"num_rec_allocs":1226073,"num_rec_compressed":0,"num_rec_compressible":0,"num_rec_frees":1226073,"num_rec_swapin"    :101377,"num_rec_swapout":110294,"num_total_smr":1,"num_trim_wctxs":2,"num_wctxs":3,"num_writers":1,"page_bytes":270790,"page_bytes_compressed":95161899,"page_bytes_marshalled":164869078,"page_c    nt":10,"page_itemcnt":2152,"persist_conflicts":2,"purges":0,"rcache_hit_ratio":0,"rcache_hits":1078475,"rcache_misses":36207,"rcompress_cache_hit_ratio":0,"rcompress_cache_hits":43770,"rcompress    _cache_misses":1034705,"reader_emptypage_cnt":0,"reader_purge_cnt":0,"reclaim_list_count":0,"reclaim_list_size":0,"reclaim_pending":0,"reclaim_threshold_mem":10485760,"reclaim_threshold_obj":50,    "reclaimed":230245162,"reclaimed_index":0,"resident_ratio":0,"rlss_num_reads":36208,"split_conflicts":0,"splits":43,"swapin_conflicts":0,"total_records":8917},"MainStore":{"act_max_delta_len":2,    "act_max_page_items":4,"act_min_page_items":1,"allocated":23229781419,"allocated_index":539036705,"avg_item_size":3068,"avg_page_size":4522,"bloom_false_positives":0,"bloom_fp_rate":0,"bloom_neg    atives":0,"bloom_tests":0,"bytes_incoming":4156210974,"cache_hit_ratio":0.5,"cache_hits":120680,"cache_misses":135,"checkpoint_used_space":0,"compact_conflicts":0,"compacts":67729,"compacts_full    marshal":48516,"compress_cache_hit_ratio":0,"compress_cache_hits":170,"compress_cache_misses":120510,"compressed_compressible_percent":0,"compressed_percent":0,"compresses":2852,"compression_rat    io":9.22831,"count":120815,"currSn":25287,"decompresses":648,"delete_conflicts":0,"deletes":0,"est_disk_size":166084338,"est_recovery_size":1533469290,"est_resident_mem":882454975,"freed":226959    26619,"freed_index":5358841,"gcSn":25285,"gcSnIntervals":"[0 25285 25286]","insert_conflicts":0,"inserts":120815,"items_count":9619,"lss_blk_rdr_reads_bs":0,"lss_blk_read_bs":698101760,"lss_data    _size":116259037,"lss_maxsn_size":4096,"lss_num_reads":37827,"lss_rdr_reads_bs":0,"lss_read_bs":685370525,"lss_recoverypt_size":77824,"memory_quota":1073741824,"memory_size":533854800,"memory_si    ze_bloom_filter":0,"memory_size_delta":603008,"memory_size_index":533677864,"merge_conflicts":0,"merges":127,"merges_emptypages":0,"mvcc_purge_ratio":1.00198,"num_active_smr":0,"num_burst_visits    ":315059,"num_compressed_pages":0,"num_free_wctxs":3,"num_pages":9422,"num_pending_smr":0,"num_periodic_visits":6086066104,"num_readers":0,"num_rec_allocs":1612110,"num_rec_compressed":0,"num_rec_compressible":0,"num_rec_frees":1612110,"num_rec_swapin":102000,"num_rec_swapout":111638,"num_total_smr":1,"num_trim_wctxs":4,"num_wctxs":3,"num_writers":1,"page_bytes":5933678,"page_bytes_com    pressed":1450977430,"page_bytes_marshalled":13390062693,"page_cnt":1312,"page_itemcnt":1934,"persist_conflicts":0,"purges":0,"rcache_hit_ratio":0,"rcache_hits":0,"rcache_misses":0,"rcompress_cac    he_hit_ratio":0,"rcompress_cache_hits":0,"rcompress_cache_misses":0,"reader_emptypage_cnt":0,"reader_purge_cnt":0,"reclaim_list_count":0,"reclaim_list_size":0,"reclaim_pending":0,"reclaim_thresh    old_mem":37035,"reclaim_threshold_obj":0,"reclaimed":22695926619,"reclaimed_index":5285873,"resident_ratio":0,"rlss_num_reads":0,"split_conflicts":0,"splits":9548,"swapin_conflicts":0,"total_records":9638}}}

      2023-06-11T22:52:19.086+00:00 indexstorage_iWatchXCustomerJourney:Idx_caseId_hitDisp_IWXCJICR_v2:11515755198712134866:1

      {"slice_0":{"BackStore":{"act_max_delta_len":30,"act_max_page_items":300,"    act_min_page_items":5,"allocated":230269614,"allocated_index":4287,"avg_item_size":125,"avg_page_size":27079,"bloom_false_positives":0,"bloom_fp_rate":0,"bloom_negatives":0,"bloom_tests":0,"byte    s_incoming":22557841,"cache_hit_ratio":0,"cache_hits":1096308,"cache_misses":36270,"checkpoint_used_space":0,"compact_conflicts":0,"compacts":5748,"compacts_fullmarshal":5177,"compress_cache_hit    _ratio":0,"compress_cache_hits":43789,"compress_cache_misses":1052519,"compressed_compressible_percent":0,"compressed_percent":0,"compresses":440,"compression_ratio":1.73248,"count":17835,"currS    n":25289,"decompresses":44208,"delete_conflicts":0,"deletes":0,"est_disk_size":28014200,"est_recovery_size":16846004,"est_resident_mem":2525565,"freed":230264519,"freed_index":0,"gcSn":25287,"gc    SnIntervals":"[0 25287 25288]","insert_conflicts":0,"inserts":17835,"items_count":8917,"lss_blk_rdr_reads_bs":42352640,"lss_blk_read_bs":51494912,"lss_data_size":840426,"lss_maxsn_size":4096,"ls    s_num_reads":36803,"lss_rdr_reads_bs":447964772,"lss_read_bs":456927345,"lss_recoverypt_size":77824,"memory_quota":1073741824,"memory_size":5095,"memory_size_bloom_filter":0,"memory_size_delta":    2816,"memory_size_index":4287,"merge_conflicts":0,"merges":0,"merges_emptypages":0,"mvcc_purge_ratio":1,"num_active_smr":0,"num_burst_visits":94244,"num_compressed_pages":0,"num_free_wctxs":3,"n    um_pages":44,"num_pending_smr":0,"num_periodic_visits":133822265,"num_readers":0,"num_rec_allocs":1226224,"num_rec_compressed":0,"num_rec_compressible":0,"num_rec_frees":1226224,"num_rec_swapin"    :101528,"num_rec_swapout":110445,"num_total_smr":1,"num_trim_wctxs":2,"num_wctxs":3,"num_writers":1,"page_bytes":270790,"page_bytes_compressed":95174250,"page_bytes_marshalled":164887053,"page_c    nt":10,"page_itemcnt":2152,"persist_conflicts":2,"purges":0,"rcache_hit_ratio":0,"rcache_hits":1078475,"rcache_misses":36268,"rcompress_cache_hit_ratio":0,"rcompress_cache_hits":43770,"rcompress    _cache_misses":1034705,"reader_emptypage_cnt":0,"reader_purge_cnt":0,"reclaim_list_count":0,"reclaim_list_size":0,"reclaim_pending":0,"reclaim_threshold_mem":10485760,"reclaim_threshold_obj":50,    "reclaimed":230264519,"reclaimed_index":0,"resident_ratio":0,"rlss_num_reads":36269,"split_conflicts":0,"splits":43,"swapin_conflicts":0,"total_records":8917},"MainStore":{"act_max_delta_len":2,    "act_max_page_items":4,"act_min_page_items":1,"allocated":23239573783,"allocated_index":539036705,"avg_item_size":10057,"avg_page_size":4971,"bloom_false_positives":0,"bloom_fp_rate":0,"bloom_ne    gatives":0,"bloom_tests":0,"bytes_incoming":4156210974,"cache_hit_ratio":0.5,"cache_hits":120680,"cache_misses":135,"checkpoint_used_space":0,"compact_conflicts":0,"compacts":67814,"compacts_ful    lmarshal":48601,"compress_cache_hit_ratio":0,"compress_cache_hits":170,"compress_cache_misses":120510,"compressed_compressible_percent":0,"compressed_percent":0,"compresses":2852,"compression_ra    tio":9.22847,"count":120815,"currSn":25289,"decompresses":648,"delete_conflicts":0,"deletes":0,"est_disk_size":166084338,"est_recovery_size":1533378609,"est_resident_mem":882454975,"freed":22705    718983,"freed_index":5358841,"gcSn":25287,"gcSnIntervals":"[0 25287 25288]","insert_conflicts":0,"inserts":120815,"items_count":9619,"lss_blk_rdr_reads_bs":0,"lss_blk_read_bs":699142144,"lss_dat    a_size":116259037,"lss_maxsn_size":4096,"lss_num_reads":37912,"lss_rdr_reads_bs":0,"lss_read_bs":686404763,"lss_recoverypt_size":77824,"memory_quota":1073741824,"memory_size":533854800,"memory_s    ize_bloom_filter":0,"memory_size_delta":603008,"memory_size_index":533677864,"merge_conflicts":0,"merges":127,"merges_emptypages":0,"mvcc_purge_ratio":1.00198,"num_active_smr":0,"num_burst_visit    s":315059,"num_compressed_pages":0,"num_free_wctxs":5,"num_pages":9422,"num_pending_smr":0,"num_periodic_visits":6097306838,"num_readers":0,"num_rec_allocs":1612110,"num_rec_compressed":0,"num_r    ec_compressible":0,"num_rec_frees":1612110,"num_rec_swapin":102000,"num_rec_swapout":111638,"num_total_smr":1,"num_trim_wctxs":1,"num_wctxs":5,"num_writers":1,"page_bytes":4747328,"page_bytes_co    mpressed":1452011158,"page_bytes_marshalled":13399840267,"page_cnt":955,"page_itemcnt":472,"persist_conflicts":0,"purges":0,"rcache_hit_ratio":0,"rcache_hits":0,"rcache_misses":0,"rcompress_cach    e_hit_ratio":0,"rcompress_cache_hits":0,"rcompress_cache_misses":0,"reader_emptypage_cnt":0,"reader_purge_cnt":0,"reclaim_list_count":0,"reclaim_list_size":0,"reclaim_pending":0,"reclaim_thresho    ld_mem":37035,"reclaim_threshold_obj":0,"reclaimed":22705718983,"reclaimed_index":5285873,"resident_ratio":0,"rlss_num_reads":0,"split_conflicts":0,"splits":9548,"swapin_conflicts":0,"total_records":9638}}}

      Attachments

        Issue Links

          Activity

            People

              akhil.mundroy Akhil Mundroy
              saptarshi.sen Saptarshi Sen
              Votes:
              0 Vote for this issue
              Watchers:
              5 Start watching this issue

              Dates

                Created:
                Updated:
                Resolved:

                PagerDuty