Move to task since this is just to help for diagnostic.

We already have an Index Stats endPoint for plasma given an instance Id. This is through plasmaDiag.

I have tested it on both dedicated and shared instance on a local setup.

a) dedicated instance (on default bucket):
 
saptarshisen@Saptarshis-MacBook-Pro n_0 % curl 127.0.0.1:9108/plasmaDiag -u Administrator:asdasd -X GET -d '{"Cmd":"listDBs"}'
---------plasma-diagnostics--------
[0] default/index01/Backstore#10910474522044442957:0 : 824707322368
[1] default/index01/Mainstore#10910474522044442957:0 : 824673251840
-------------------------------------
saptarshisen@Saptarshis-MacBook-Pro n_0 % curl 127.0.0.1:9108/plasmaDiag -u Administrator:asdasd -X GET -d '{"Cmd":"stats", "Args":[824707322368]}'
---------plasma-diagnostics--------
{
"memory_quota": 241591910,
"count": 100000,
"compacts": 3030,
"purges": 0,
"splits": 499,
"merges": 0,
"inserts": 100000,
"deletes": 0,
"compact_conflicts": 323,
"split_conflicts": 173,
"merge_conflicts": 0,
"insert_conflicts": 597,
"delete_conflicts": 0,
"swapin_conflicts": 0,
"persist_conflicts": 0,
"memory_size": 14056788,
"memory_size_index": 50512,
"allocated": 113521303,
"freed": 99464515,
"reclaimed": 99464515,
"reclaim_pending": 0,
"reclaim_list_size": 0,
"reclaim_list_count": 0,
"reclaim_threshold": 36,
"allocated_index": 50512,
"freed_index": 0,
"reclaimed_index": 0,
"num_pages": 500,
"items_count": 100000,
"total_records": 111719,
"num_rec_allocs": 894178,
"num_rec_frees": 782459,
"num_rec_swapout": 0,
"num_rec_swapin": 0,
"num_rec_compressed": 0,
"compresses": 0,
"decompresses": 0,
"num_compressed_pages": 0,
"bytes_incoming": 11477884,
"lss_data_size": 5200027,
"lss_recoverypt_size": 8192,
"lss_maxsn_size": 4096,
"checkpoint_used_space":0,
"est_disk_size": 14444519,
"est_recovery_size": 276332,
"lss_num_reads": 0,
"lss_read_bs": 0,
"lss_blk_read_bs": 0,
"lss_rdr_reads_bs": 0,
"lss_blk_rdr_reads_bs": 0,
"cache_hits": 200000,
"cache_misses": 0,
"cache_hit_ratio": 1.00000,
"rlss_num_reads": 0,
"rcache_hits": 100000,
"rcache_misses": 0,
"rcache_hit_ratio": 1.00000,
"resident_ratio": 1.00000,
"mvcc_purge_ratio": 1.11719,
"currSn": 10122,
"gcSn": 10120,
"gcSnIntervals": "[0 2 10121]",
"purger_running": false,
"mem_throttled": false,
"lss_throttled": false,
"num_wctxs": 6,
"num_free_wctxs": 7,
"num_readers": 0,
"num_writers": 2,
"page_bytes": 7471943,
"page_cnt": 291,
"page_itemcnt": 65098,
"avg_item_size": 114,
"avg_page_size": 25676,
"act_max_page_items": 300,
"act_min_page_items": 5,
"act_max_delta_len": 30,
"est_resident_mem": 14107300,
"page_bytes_marshalled":26438915,
"page_bytes_compressed":11890608,
"compression_ratio": 2.22351,
"bloom_tests": 0,
"bloom_negatives": 0,
"bloom_false_positives":0,
"bloom_fp_rate": 0.00000,
"memory_size_bloom_filter":0,
"memory_size_delta": 366400,
"num_total_smr": 1,
"num_active_smr": 0,
"num_pending_smr": 0,
"lss_stats":

}-------------------------------------
 b) shared instance (on a collection):

 
saptarshisen@Saptarshis-MacBook-Pro data % curl 127.0.0.1:9108/plasmaDiag -u Administrator:asdasd -X GET -d '{"Cmd":"listDBs"}'
---------plasma-diagnostics--------
[0] ssen/index_flightid/Backstore#5204233635097458129:0 : 824673805824
[1] ssen/index_nameid/Backstore#18177150202641995915:0 : 824673250304
[2] ssen/index_flightid/Mainstore#5204233635097458129:0 : 824707714048
[3] ssen/index_nameid/Mainstore#18177150202641995915:0 : 824714605056
 
saptarshisen@Saptarshis-MacBook-Pro data % curl 127.0.0.1:9108/plasmaDiag -u Administrator:asdasd -X GET -d '{"Cmd":"stats", "Args":[824673805824]}'
---------plasma-diagnostics--------
{
"memory_quota": 241591910,
"count": 100000,
"compacts": 3028,
"purges": 0,
"splits": 506,
"merges": 0,
"inserts": 100000,
"deletes": 0,
"compact_conflicts": 260,
"split_conflicts": 152,
"merge_conflicts": 0,
"insert_conflicts": 496,
"delete_conflicts": 0,
"swapin_conflicts": 0,
"persist_conflicts": 0,
"memory_size": 10698543,
"memory_size_index": 51392,
"allocated": 86531007,
"freed": 75832464,
"reclaimed": 73426656,
"reclaim_pending": 2405808,
"reclaim_list_size": 2405808,
"reclaim_list_count": 102,
"reclaim_threshold": 39,
"allocated_index": 51392,
"freed_index": 0,
"reclaimed_index": 0,
"num_pages": 507,
"items_count": 100000,
"total_records": 111876,
"num_rec_allocs": 894538,
"num_rec_frees": 782662,
"num_rec_swapout": 0,
"num_rec_swapin": 0,
"num_rec_compressed": 0,
"compresses": 0,
"decompresses": 0,
"num_compressed_pages": 0,
"bytes_incoming": 8455380,
"lss_data_size": 5118535,
"lss_recoverypt_size": 4096,
"lss_maxsn_size": 4096,
"checkpoint_used_space":0,
"est_disk_size": 14218152,
"est_recovery_size": 269040,
"lss_num_reads": 0,
"lss_read_bs": 0,
"lss_blk_read_bs": 0,
"lss_rdr_reads_bs": 0,
"lss_blk_rdr_reads_bs": 0,
"cache_hits": 100000,
"cache_misses": 0,
"cache_hit_ratio": 1.00000,
"rlss_num_reads": 0,
"rcache_hits": 0,
"rcache_misses": 0,
"rcache_hit_ratio": 0.00000,
"resident_ratio": 1.00000,
"mvcc_purge_ratio": 1.11876,
"currSn": 2,
"gcSn": 0,
"gcSnIntervals": "[0 1]",
"purger_running": false,
"mem_throttled": false,
"lss_throttled": false,
"num_wctxs": 5,
"num_free_wctxs": 8,
"num_readers": 0,
"num_writers": 4,
"page_bytes": 7853015,
"page_cnt": 408,
"page_itemcnt": 92868,
"avg_item_size": 84,
"avg_page_size": 19247,
"act_max_page_items": 300,
"act_min_page_items": 5,
"act_max_delta_len": 30,
"est_resident_mem": 10749935,
"page_bytes_marshalled":19688764,
"page_bytes_compressed":11808847,
"compression_ratio": 1.66729,
"bloom_tests": 0,
"bloom_negatives": 0,
"bloom_false_positives":0,
"bloom_fp_rate": 0.00000,
"memory_size_bloom_filter":0,
"memory_size_delta": 370608,
"num_total_smr": 1,
"num_active_smr": 0,
"num_pending_smr": 0,
"lss_stats":

}-------------------------------------

Is it possible to log a specific index to the log periodically?

Yes, i have attached a sample output from my testing. Thanks

Hi John,

     Based on review comments,
>>For MB-43997, indexer would periodically log plasma stats.  But that stats is filtered out for some instance. For debugging purpose, we want to some instance to NOT filter out.
  
    I have updated the patch accordingly, which can disable this filtering by StatsLogger and allows logging for all plasma indexes.
    http://review.couchbase.org/c/plasma/+/157744 (P3)
 
    Sample usage to disable filtering:
 
{{   curl -i 127.0.0.1:9102/plasmaDiag -u Administrator:asdasd -X GET -d '

'        }}
    HTTP/1.1 200 OK
 
{{}}Testing Steps :
 
    a) Modified the indexer config to keep a high percentile for threshold. (indexer_settings.txt)
 
    b) uploaded the travel-sample bucket
 
curl -i 127.0.0.1:9102/plasmaDiag -u Administrator:asdasd -X GET -d '{"Cmd":"listDBs"}'
HTTP/1.1 200 OK
Date: Wed, 21 Jul 2021 15:52:26 GMT
Content-Length: 880
Content-Type: text/plain; charset=utf-8
 
[0] travel-sample/def_name_type/Mainstore#9564981566735079727:0: 824718612480
[1] travel-sample/def_name_type/Backstore#9564981566735079727:0: 824729100288
[2] travel-sample/def_airportname/Backstore#3038861201680102476:0: 824762797056
[3] travel-sample/def_route_src_dst_day/Backstore#17647123905126878788:0: 824729101824
[4] travel-sample/def_airportname/Mainstore#3038861201680102476:0: 824750296064
[5] travel-sample/def_route_src_dst_day/Mainstore#17647123905126878788:0: 824750984192
[6] travel-sample/def_inventory_route_route_src_dst_day/Backstore#724971241206167540:0: 824718607872
[7] travel-sample/def_inventory_route_route_src_dst_day/Mainstore#724971241206167540:0: 824759315456
[8] travel-sample/def_inventory_airport_primary/Mainstore#18111893483924174824:0: 824718606336
[9] travel-sample/def_inventory_hotel_primary/Mainstore#12030138103339116414:0: 824767171584
 
    c) Monitor the Storage-Stats in indexer.log:
 
2021-07-21T08:51:16.290-07:00 [Info]
==== StorageStats ====
[
{{{}}
"Index": "travel-sample:inventory:hotel:def_inventory_hotel_primary", "Id": 12030138103339116414, "PartitionId": 0,
"Stats":
{{{}}
}
}
,{
"Index": "travel-sample:def_name_type", "Id": 9564981566735079727, "PartitionId": 0,
"Stats":
{{{}}
}
}
,{
"Index": "travel-sample:def_airportname", "Id": 3038861201680102476, "PartitionId": 0,
"Stats":
{{{}}
}
}
,{
"Index": "travel-sample:def_route_src_dst_day", "Id": 17647123905126878788, "PartitionId": 0,
"Stats":
{{{}}
}
}
,{
"Index": "travel-sample:inventory:route:def_inventory_route_route_src_dst_day", "Id": 724971241206167540, "PartitionId": 0,
"Stats":
{{{}}
}
}
,{
"Index": "travel-sample:inventory:airport:def_inventory_airport_primary", "Id": 18111893483924174824, "PartitionId": 0,
"Stats":
{{{}}
}
}
] 
d) Disable filtering
 
curl -i 127.0.0.1:9102/plasmaDiag -u Administrator:asdasd -X GET -d '{"Cmd":"filterindexstats", "Args":[false]}'        
HTTP/1.1 200 OK
Date: Wed, 21 Jul 2021 15:53:15 GMT
Content-Length: 0
 
e) Monitor indexer.log (attached storagestats_nofilter.txt)
 
2021-07-21T08:54:16.296-07:00 [Info]
==== StorageStats ====
[
{{{}}
"Index": "travel-sample:def_route_src_dst_day", "Id": 17647123905126878788, "PartitionId": 0,
"Stats":
{{{}}
"MainStore":
{{{}}
"memory_quota": 966367641,
"count": 168168,
"compacts": 1188,
"purges": 0,
"splits": 577,
"merges": 0,
"inserts": 168168,
"deletes": 0,
"compact_conflicts": 93,
"split_conflicts": 114,
"merge_conflicts": 0,
"insert_conflicts": 991,
"delete_conflicts": 0,
"swapin_conflicts": 0,
"persist_conflicts": 0,
"memory_size": 10499277,
"memory_size_index": 51976,
"allocated": 41336425,
"freed": 30837148,
"reclaimed": 27791334,
"reclaim_pending": 3045814,
"reclaim_list_size": 3045814,
"reclaim_list_count": 119,
"reclaim_threshold": 18,
"allocated_index": 51976,
"freed_index": 0,
"reclaimed_index": 0,
"num_pages": 578,
"items_count": 168168,
"total_records": 168168,
"num_rec_allocs": 669745,
"num_rec_frees": 501577,
"num_rec_swapout": 0,
"num_rec_swapin": 0,
"num_rec_compressed": 0,
"compresses": 0,
"decompresses": 0,
"num_compressed_pages": 0,
"bytes_incoming": 7607453,
"lss_data_size": 937960,
"lss_recoverypt_size": 4096,
"lss_maxsn_size": 4096,
"checkpoint_used_space":0,
"est_disk_size": 1914204,
"est_recovery_size": 164876,
"lss_num_reads": 0,
"lss_read_bs": 0,
"lss_blk_read_bs": 0,
"lss_rdr_reads_bs": 0,
"lss_blk_rdr_reads_bs": 0,
"cache_hits": 168168,
"cache_misses": 0,
"cache_hit_ratio": 1.00000,
"rlss_num_reads": 0,
"rcache_hits": 0,
"rcache_misses": 0,
"rcache_hit_ratio": 0.00000,
"resident_ratio": 1.00000,
"mvcc_purge_ratio": 1.00000,
"currSn": 286,
"gcSn": 284,
"gcSnIntervals": "[0 285]",
"purger_running": false,
"mem_throttled": false,
"lss_throttled": false,
"num_wctxs": 4,
"num_free_wctxs": 15,
"num_readers": 0,
"num_writers": 12,
"page_bytes": 2767472,
"page_cnt": 209,
"page_itemcnt": 61160,
"avg_item_size": 45,
"avg_page_size": 13241,
"act_max_page_items": 400,
"act_min_page_items": 25,
"act_max_delta_len": 200,
"est_resident_mem": 10551253,
"page_bytes_marshalled":11164664,
"page_bytes_compressed":1963523,
"compression_ratio": 5.68604,
"bloom_tests": 0,
"bloom_negatives": 0,
"bloom_false_positives":0,
"bloom_fp_rate": 0.00000,
"memory_size_bloom_filter":0,
"memory_size_delta": 1883312,
"num_total_smr": 1,
"num_active_smr": 0,
"num_pending_smr": 0,
"lss_stats":
{{{}}
"shared": false,
"punch_hole_support": false,
"buf_memused": 0,
"lss_data_size": 1017661,
"lss_used_space": 2105344,
"lss_disk_size": 2105344,
"lss_fragmentation": 51,
"lss_num_reads": 0,
"lss_read_bs": 0,
"lss_blk_read_bs": 0,
"bytes_written": 2105344,
"bytes_incoming": 7607453,
"write_amp": 0.28,
"write_amp_avg": 0.28,
"lss_gc_num_reads": 0,
"lss_gc_reads_bs": 0,
"lss_blk_gc_reads_bs": 0,
"lss_gc_status": "",
"lss_head_offset": 0,
"lss_tail_offset": 2101248,
"num_sctxs": 13,
"num_free_sctxs": 5,
"num_smr_worker ": 12,
"active_smr_worker ": 0,
"num_smr_work_item": 0,
"num_swapperWriter": 24
}
},
"BackStore":
{{{}}
"memory_quota": 966367641,
"count": 24024,
"compacts": 841,
"purges": 0,
"splits": 125,
"merges": 0,
"inserts": 24024,
"deletes": 0,
"compact_conflicts": 162,
"split_conflicts": 79,
"merge_conflicts": 0,
"insert_conflicts": 233,
"delete_conflicts": 0,
"swapin_conflicts": 0,
"persist_conflicts": 0,
"memory_size": 4546824,
"memory_size_index": 8349,
"allocated": 44300762,
"freed": 39753938,
"reclaimed": 39753938,
"reclaim_pending": 0,
"reclaim_list_size": 0,
"reclaim_list_count": 0,
"reclaim_threshold": 12,
"allocated_index": 8349,
"freed_index": 0,
"reclaimed_index": 0,
"num_pages": 126,
"items_count": 24024,
"total_records": 24024,
"num_rec_allocs": 233477,
"num_rec_frees": 209453,
"num_rec_swapout": 0,
"num_rec_swapin": 0,
"num_rec_compressed": 0,
"compresses": 0,
"decompresses": 0,
"num_compressed_pages": 0,
"bytes_incoming": 4277331,
"lss_data_size": 301580,
"lss_recoverypt_size": 4096,
"lss_maxsn_size": 4096,
"checkpoint_used_space":0,
"est_disk_size": 2010533,
"est_recovery_size": 30794,
"lss_num_reads": 0,
"lss_read_bs": 0,
"lss_blk_read_bs": 0,
"lss_rdr_reads_bs": 0,
"lss_blk_rdr_reads_bs": 0,
"cache_hits": 55615,
"cache_misses": 0,
"cache_hit_ratio": 1.00000,
"rlss_num_reads": 0,
"rcache_hits": 31591,
"rcache_misses": 0,
"rcache_hit_ratio": 1.00000,
"resident_ratio": 1.00000,
"mvcc_purge_ratio": 1.00000,
"currSn": 286,
"gcSn": 284,
"gcSnIntervals": "[0 285]",
"purger_running": false,
"mem_throttled": false,
"lss_throttled": false,
"num_wctxs": 4,
"num_free_wctxs": 4,
"num_readers": 0,
"num_writers": 1,
"page_bytes": 4921367,
"page_cnt": 126,
"page_itemcnt": 27525,
"avg_item_size": 178,
"avg_page_size": 39058,
"act_max_page_items": 300,
"act_min_page_items": 5,
"act_max_delta_len": 30,
"est_resident_mem": 4555173,
"page_bytes_marshalled":6877830,
"page_bytes_compressed":1523794,
"compression_ratio": 4.51362,
"bloom_tests": 0,
"bloom_negatives": 0,
"bloom_false_positives":0,
"bloom_fp_rate": 0.00000,
"memory_size_bloom_filter":0,
"memory_size_delta": 90512,
"num_total_smr": 1,
"num_active_smr": 0,
"num_pending_smr": 0,
"lss_stats":
{{{}}
"shared": false,
"punch_hole_support": false,
"buf_memused": 0,
"lss_data_size": 313757,
"lss_used_space": 2093056,
"lss_disk_size": 2093056,
"lss_fragmentation": 85,
"lss_num_reads": 0,
"lss_read_bs": 0,
"lss_blk_read_bs": 0,
"bytes_written": 2093056,
"bytes_incoming": 4277331,
"write_amp": 0.49,
"write_amp_avg": 0.49,
"lss_gc_num_reads": 0,
"lss_gc_reads_bs": 0,
"lss_blk_gc_reads_bs": 0,
"lss_gc_status": "",
"lss_head_offset": 0,
"lss_tail_offset": 2088960,
"num_sctxs": 13,
"num_free_sctxs": 5,
"num_smr_worker ": 12,
"active_smr_worker ": 0,
"num_smr_work_item": 0,
"num_swapperWriter": 24
}
}}
}
.......
,
}}
}
]
 
 Please let me know, if this is what we need.
 
Also, during the course of testing, I came across an issue in StatsLogger.run.  I have created an MB for thishttps://issues.couchbase.com/browse/MB-47530
 

Build couchbase-server-7.1.0-1138 contains plasma commit ba378fd with commit message:
MB-43997: Add filterStatsLogging Cmd to plasma diag

Build couchbase-server-7.1.0-1138 contains plasma commit ba378fd with commit message:
MB-43997: Add filterStatsLogging Cmd to plasma diag

Used curl command for plasmaDiag endpoint, to enable & the disable the filtering and looked at the indexer.log