Details
-
Bug
-
Resolution: Not a Bug
-
Critical
-
6.5.0
-
Untriaged
-
-
Yes
Description
Build: 6.5.0-4471
Test:
./testrunner -i /tmp/testexec.30071.ini -p reset_services=True,use_gsi_for_primary=True,defer_build=True,initial_index_number=2,doc-per-day=10,reset_services=True,use_gsi_for_primary=True,defer_build=True,initial_index_number=2,doc-per-day=10 -t 2i.memdb_oom_2i.SecondaryIndexMemdbOomTests.test_oom_delete_bucket,gsi_type=memory_optimized,nodes_init=4,services_init=kv:n1ql-kv-index-index,doc_ops=True,standard_buckets=2,default_buckets=1,initial_index_number=4,scan_consistency=request_plus
Steps:
1) Load data to multiple buckets, bucket1, bucket2 etc
2) Create multiple indexes for each of the buckets like index0, index1, index2 etc with MOI storage mode
3) perform enough mutations so we achieve OOM and indexer_state become paused.
4) delete a bucket so indexes w.r.t bucket also removed.
5) Even after this, other indexes are not turned to Active
Expectation is, once there is enough room, paused indexes should turn Active.
stats for reference:
curl -XGET 172.23.109.181:9102/stats -u Administrator:password | jq
|
% Total % Received % Xferd Average Speed Time Time Time Current
|
Dload Upload Total Spent Left Speed
|
100 11097 0 11097 0 0 248k 0 --:--:-- --:--:-- --:--:-- 252k
|
{
|
"17176740282792272409:completion_progress": 4636737291354636000,
|
"1929713365505122981:completion_progress": 4636737291354636000,
|
"MAINT_STREAM/172.23.109.172:8091/projector_latency": 13510126,
|
"MAINT_STREAM/172.23.109.179:8091/projector_latency": 12298740,
|
"cpu_utilization": 8.63275781614559,
|
"index_not_found_errcount": 0,
|
"indexer_state": "Paused",
|
"memory_free": 1445535744,
|
"memory_quota": 268435456,
|
"memory_rss": 214044672,
|
"memory_total": 3867238400,
|
"memory_total_storage": 136122368,
|
"memory_used": 233549824,
|
"memory_used_queue": 2510,
|
"memory_used_storage": 46579010,
|
"needs_restart": false,
|
"num_connections": 3,
|
"num_cpu_core": 4,
|
"standard_bucket0:index_name_0:avg_disk_bps": 0,
|
"standard_bucket0:index_name_0:avg_drain_rate": 0,
|
"standard_bucket0:index_name_0:avg_mutation_rate": 0,
|
"standard_bucket0:index_name_0:avg_scan_latency": 0,
|
"standard_bucket0:index_name_0:avg_scan_rate": 0,
|
"standard_bucket0:index_name_0:avg_scan_request_alloc_latency": 0,
|
"standard_bucket0:index_name_0:avg_scan_request_init_latency": 0,
|
"standard_bucket0:index_name_0:avg_scan_request_latency": 0,
|
"standard_bucket0:index_name_0:avg_scan_wait_latency": 0,
|
"standard_bucket0:index_name_0:avg_ts_interval": 45302799,
|
"standard_bucket0:index_name_0:avg_ts_items_count": 1,
|
"standard_bucket0:index_name_0:backstore_data_size": 3133506,
|
"standard_bucket0:index_name_0:build_progress": 100,
|
"standard_bucket0:index_name_0:cache_hit_percent": 100,
|
"standard_bucket0:index_name_0:cache_hits": 0,
|
"standard_bucket0:index_name_0:cache_misses": 0,
|
"standard_bucket0:index_name_0:client_cancel_errcount": 0,
|
"standard_bucket0:index_name_0:data_size": 17877769,
|
"standard_bucket0:index_name_0:delete_bytes": 0,
|
"standard_bucket0:index_name_0:disk_load_duration": 0,
|
"standard_bucket0:index_name_0:disk_size": 17240465,
|
"standard_bucket0:index_name_0:disk_store_duration": 102,
|
"standard_bucket0:index_name_0:flush_queue_size": 0,
|
"standard_bucket0:index_name_0:frag_percent": 0,
|
"standard_bucket0:index_name_0:get_bytes": 0,
|
"standard_bucket0:index_name_0:insert_bytes": 0,
|
"standard_bucket0:index_name_0:items_count": 87827,
|
"standard_bucket0:index_name_0:key_size_distribution": {
|
"(0-64)": 0,
|
"(102401-max)": 0,
|
"(1025-4096)": 0,
|
"(257-1024)": 0,
|
"(4097-102400)": 0,
|
"(65-256)": 87827
|
},
|
"standard_bucket0:index_name_0:key_size_stats_since": 1570659450719464700,
|
"standard_bucket0:index_name_0:last_known_scan_time": 0,
|
"standard_bucket0:index_name_0:last_rollback_time": "1570659448155362546",
|
"standard_bucket0:index_name_0:memory_used": 22763481,
|
"standard_bucket0:index_name_0:not_ready_errcount": 0,
|
"standard_bucket0:index_name_0:num_commits": 3,
|
"standard_bucket0:index_name_0:num_compactions": 0,
|
"standard_bucket0:index_name_0:num_completed_requests": 0,
|
"standard_bucket0:index_name_0:num_completed_requests_aggr": 0,
|
"standard_bucket0:index_name_0:num_completed_requests_range": 0,
|
"standard_bucket0:index_name_0:num_docs_indexed": 89579,
|
"standard_bucket0:index_name_0:num_docs_pending": 0,
|
"standard_bucket0:index_name_0:num_docs_processed": 89579,
|
"standard_bucket0:index_name_0:num_docs_queued": 13237,
|
"standard_bucket0:index_name_0:num_flush_queued": 89579,
|
"standard_bucket0:index_name_0:num_items_flushed": 89579,
|
"standard_bucket0:index_name_0:num_items_restored": 0,
|
"standard_bucket0:index_name_0:num_last_snapshot_reply": 0,
|
"standard_bucket0:index_name_0:num_open_snapshots": 1,
|
"standard_bucket0:index_name_0:num_requests": 0,
|
"standard_bucket0:index_name_0:num_requests_aggr": 0,
|
"standard_bucket0:index_name_0:num_requests_range": 0,
|
"standard_bucket0:index_name_0:num_rows_returned": 0,
|
"standard_bucket0:index_name_0:num_rows_returned_aggr": 0,
|
"standard_bucket0:index_name_0:num_rows_returned_range": 0,
|
"standard_bucket0:index_name_0:num_rows_scanned": 0,
|
"standard_bucket0:index_name_0:num_rows_scanned_aggr": 0,
|
"standard_bucket0:index_name_0:num_rows_scanned_range": 0,
|
"standard_bucket0:index_name_0:num_scan_errors": 0,
|
"standard_bucket0:index_name_0:num_scan_timeouts": 0,
|
"standard_bucket0:index_name_0:num_snapshot_waiters": 0,
|
"standard_bucket0:index_name_0:num_snapshots": 63233,
|
"standard_bucket0:index_name_0:progress_stat_time": "1570666079285875004",
|
"standard_bucket0:index_name_0:recs_in_mem": 87827,
|
"standard_bucket0:index_name_0:recs_on_disk": 0,
|
"standard_bucket0:index_name_0:resident_percent": 100,
|
"standard_bucket0:index_name_0:scan_bytes_read": 0,
|
"standard_bucket0:index_name_0:scan_cache_hit_aggr": 0,
|
"standard_bucket0:index_name_0:scan_cache_hit_range": 0,
|
"standard_bucket0:index_name_0:scan_wait_duration": 0,
|
"standard_bucket0:index_name_0:since_last_snapshot": 75282925,
|
"standard_bucket0:index_name_0:timings/dcp_getseqs": "0 0 0",
|
"standard_bucket0:index_name_0:timings/n1ql_expr_eval": "0 0 0",
|
"standard_bucket0:index_name_0:timings/scan_pipeline_iterate": "0 0 0",
|
"standard_bucket0:index_name_0:timings/storage_clone_handle": "0 0 0",
|
"standard_bucket0:index_name_0:timings/storage_commit": "0 0 0",
|
"standard_bucket0:index_name_0:timings/storage_del": "0 0 0",
|
"standard_bucket0:index_name_0:timings/storage_get": "0 0 0",
|
"standard_bucket0:index_name_0:timings/storage_info": "0 0 0",
|
"standard_bucket0:index_name_0:timings/storage_iterator_next": "0 0 0",
|
"standard_bucket0:index_name_0:timings/storage_meta_get": "0 0 0",
|
"standard_bucket0:index_name_0:timings/storage_meta_set": "0 0 0",
|
"standard_bucket0:index_name_0:timings/storage_new_iterator": "0 0 0",
|
"standard_bucket0:index_name_0:timings/storage_persist_snapshot_create": "0 0 0",
|
"standard_bucket0:index_name_0:timings/storage_set": "89579 2813035826 127759474790704",
|
"standard_bucket0:index_name_0:timings/storage_snapshot_close": "0 0 0",
|
"standard_bucket0:index_name_0:timings/storage_snapshot_create": "0 0 0",
|
"standard_bucket0:index_name_0:total_scan_duration": 0,
|
"standard_bucket0:index_name_0:total_scan_request_duration": 0,
|
"standard_bucket0:index_name_1:avg_disk_bps": 0,
|
"standard_bucket0:index_name_1:avg_drain_rate": 0,
|
"standard_bucket0:index_name_1:avg_mutation_rate": 0,
|
"standard_bucket0:index_name_1:avg_scan_latency": 0,
|
"standard_bucket0:index_name_1:avg_scan_rate": 0,
|
"standard_bucket0:index_name_1:avg_scan_request_alloc_latency": 0,
|
"standard_bucket0:index_name_1:avg_scan_request_init_latency": 0,
|
"standard_bucket0:index_name_1:avg_scan_request_latency": 0,
|
"standard_bucket0:index_name_1:avg_scan_wait_latency": 0,
|
"standard_bucket0:index_name_1:avg_ts_interval": 45707528,
|
"standard_bucket0:index_name_1:avg_ts_items_count": 1,
|
"standard_bucket0:index_name_1:backstore_data_size": 3133506,
|
"standard_bucket0:index_name_1:build_progress": 100,
|
"standard_bucket0:index_name_1:cache_hit_percent": 100,
|
"standard_bucket0:index_name_1:cache_hits": 0,
|
"standard_bucket0:index_name_1:cache_misses": 0,
|
"standard_bucket0:index_name_1:client_cancel_errcount": 0,
|
"standard_bucket0:index_name_1:data_size": 17877769,
|
"standard_bucket0:index_name_1:delete_bytes": 0,
|
"standard_bucket0:index_name_1:disk_load_duration": 0,
|
"standard_bucket0:index_name_1:disk_size": 17240468,
|
"standard_bucket0:index_name_1:disk_store_duration": 105,
|
"standard_bucket0:index_name_1:flush_queue_size": 0,
|
"standard_bucket0:index_name_1:frag_percent": 0,
|
"standard_bucket0:index_name_1:get_bytes": 0,
|
"standard_bucket0:index_name_1:insert_bytes": 0,
|
"standard_bucket0:index_name_1:items_count": 87827,
|
"standard_bucket0:index_name_1:key_size_distribution": {
|
"(0-64)": 0,
|
"(102401-max)": 0,
|
"(1025-4096)": 0,
|
"(257-1024)": 0,
|
"(4097-102400)": 0,
|
"(65-256)": 87827
|
},
|
"standard_bucket0:index_name_1:key_size_stats_since": 1570659458167555000,
|
"standard_bucket0:index_name_1:last_known_scan_time": 0,
|
"standard_bucket0:index_name_1:last_rollback_time": "1570659448155362546",
|
"standard_bucket0:index_name_1:memory_used": 22766953,
|
"standard_bucket0:index_name_1:not_ready_errcount": 0,
|
"standard_bucket0:index_name_1:num_commits": 3,
|
"standard_bucket0:index_name_1:num_compactions": 0,
|
"standard_bucket0:index_name_1:num_completed_requests": 0,
|
"standard_bucket0:index_name_1:num_completed_requests_aggr": 0,
|
"standard_bucket0:index_name_1:num_completed_requests_range": 0,
|
"standard_bucket0:index_name_1:num_docs_indexed": 89579,
|
"standard_bucket0:index_name_1:num_docs_pending": 0,
|
"standard_bucket0:index_name_1:num_docs_processed": 89579,
|
"standard_bucket0:index_name_1:num_docs_queued": 13237,
|
"standard_bucket0:index_name_1:num_flush_queued": 89579,
|
"standard_bucket0:index_name_1:num_items_flushed": 89579,
|
"standard_bucket0:index_name_1:num_items_restored": 0,
|
"standard_bucket0:index_name_1:num_last_snapshot_reply": 0,
|
"standard_bucket0:index_name_1:num_open_snapshots": 1,
|
"standard_bucket0:index_name_1:num_requests": 0,
|
"standard_bucket0:index_name_1:num_requests_aggr": 0,
|
"standard_bucket0:index_name_1:num_requests_range": 0,
|
"standard_bucket0:index_name_1:num_rows_returned": 0,
|
"standard_bucket0:index_name_1:num_rows_returned_aggr": 0,
|
"standard_bucket0:index_name_1:num_rows_returned_range": 0,
|
"standard_bucket0:index_name_1:num_rows_scanned": 0,
|
"standard_bucket0:index_name_1:num_rows_scanned_aggr": 0,
|
"standard_bucket0:index_name_1:num_rows_scanned_range": 0,
|
"standard_bucket0:index_name_1:num_scan_errors": 0,
|
"standard_bucket0:index_name_1:num_scan_timeouts": 0,
|
"standard_bucket0:index_name_1:num_snapshot_waiters": 0,
|
"standard_bucket0:index_name_1:num_snapshots": 63236,
|
"standard_bucket0:index_name_1:progress_stat_time": "1570666079285881814",
|
"standard_bucket0:index_name_1:recs_in_mem": 87827,
|
"standard_bucket0:index_name_1:recs_on_disk": 0,
|
"standard_bucket0:index_name_1:resident_percent": 100,
|
"standard_bucket0:index_name_1:scan_bytes_read": 0,
|
"standard_bucket0:index_name_1:scan_cache_hit_aggr": 0,
|
"standard_bucket0:index_name_1:scan_cache_hit_range": 0,
|
"standard_bucket0:index_name_1:scan_wait_duration": 0,
|
"standard_bucket0:index_name_1:since_last_snapshot": 76710424,
|
"standard_bucket0:index_name_1:timings/dcp_getseqs": "0 0 0",
|
"standard_bucket0:index_name_1:timings/n1ql_expr_eval": "0 0 0",
|
"standard_bucket0:index_name_1:timings/scan_pipeline_iterate": "0 0 0",
|
"standard_bucket0:index_name_1:timings/storage_clone_handle": "0 0 0",
|
"standard_bucket0:index_name_1:timings/storage_commit": "0 0 0",
|
"standard_bucket0:index_name_1:timings/storage_del": "0 0 0",
|
"standard_bucket0:index_name_1:timings/storage_get": "0 0 0",
|
"standard_bucket0:index_name_1:timings/storage_info": "0 0 0",
|
"standard_bucket0:index_name_1:timings/storage_iterator_next": "0 0 0",
|
"standard_bucket0:index_name_1:timings/storage_meta_get": "0 0 0",
|
"standard_bucket0:index_name_1:timings/storage_meta_set": "0 0 0",
|
"standard_bucket0:index_name_1:timings/storage_new_iterator": "0 0 0",
|
"standard_bucket0:index_name_1:timings/storage_persist_snapshot_create": "0 0 0",
|
"standard_bucket0:index_name_1:timings/storage_set": "89579 2721996443 115180709445577",
|
"standard_bucket0:index_name_1:timings/storage_snapshot_close": "0 0 0",
|
"standard_bucket0:index_name_1:timings/storage_snapshot_create": "0 0 0",
|
"standard_bucket0:index_name_1:total_scan_duration": 0,
|
"standard_bucket0:index_name_1:total_scan_request_duration": 0,
|
"standard_bucket0:mutation_queue_size": 5,
|
"standard_bucket0:num_mutations_queued": 91600,
|
"standard_bucket0:num_nonalign_ts": 1,
|
"standard_bucket0:num_rollbacks": 0,
|
"standard_bucket0:timings/dcp_getseqs": "1112 7424413965 830656251533192403",
|
"standard_bucket0:ts_queue_size": 0,
|
"storage_mode": "memory_optimized",
|
"timestamp": "1570666083313046816",
|
"timings/stats_response": "9571 25104760965 240451921038949027",
|
"uptime": "1h53m52.148832347s"
|
}
|
curl -XGET 172.23.109.181:9102/stats/storage -u Administrator:password | jq
|
% Total % Received % Xferd Average Speed Time Time Time Current
|
Dload Upload Total Spent Left Speed
|
100 2657 0 2657 0 0 53710 0 --:--:-- --:--:-- --:--:-- 54224
|
[
|
{
|
"Index": "standard_bucket0:index_name_1",
|
"Id": 17176740282792272000,
|
"PartitionId": 0,
|
"Stats": {
|
"MainStore": {
|
"node_count": 87827,
|
"soft_deletes": 0,
|
"read_conflicts": 0,
|
"insert_conflicts": 218,
|
"next_pointers_per_node": 1.3341,
|
"memory_used": 19078155,
|
"node_allocs": 87827,
|
"node_frees": 0,
|
"level_node_distribution": {
|
"level0": 65748,
|
"level1": 16622,
|
"level2": 4131,
|
"level3": 966,
|
"level4": 272,
|
"level5": 66,
|
"level6": 11,
|
"level7": 10,
|
"level8": 1,
|
"level9": 0,
|
"level10": 0,
|
"level11": 0,
|
"level12": 0,
|
"level13": 0,
|
"level14": 0,
|
"level15": 0,
|
"level16": 0,
|
"level17": 0,
|
"level18": 0,
|
"level19": 0,
|
"level20": 0,
|
"level21": 0,
|
"level22": 0,
|
"level23": 0,
|
"level24": 0,
|
"level25": 0,
|
"level26": 0,
|
"level27": 0,
|
"level28": 0,
|
"level29": 0,
|
"level30": 0,
|
"level31": 0,
|
"level32": 0
|
}
|
},
|
"BackStore_0": {
|
"FastHTCount": 21984,
|
"SlowHTCount": 0,
|
"Conflicts": 0,
|
"MemoryInUse": 923328
|
},
|
"BackStore_1": {
|
"FastHTCount": 21934,
|
"SlowHTCount": 0,
|
"Conflicts": 0,
|
"MemoryInUse": 921228
|
},
|
"BackStore_2": {
|
"FastHTCount": 21938,
|
"SlowHTCount": 0,
|
"Conflicts": 0,
|
"MemoryInUse": 921396
|
},
|
"BackStore_3": {
|
"FastHTCount": 21971,
|
"SlowHTCount": 0,
|
"Conflicts": 0,
|
"MemoryInUse": 922782
|
},
|
"data_size": 19078219,
|
"items_count": 87827
|
}
|
},
|
{
|
"Index": "standard_bucket0:index_name_0",
|
"Id": 1929713365505123000,
|
"PartitionId": 0,
|
"Stats": {
|
"MainStore": {
|
"node_count": 87827,
|
"soft_deletes": 0,
|
"read_conflicts": 0,
|
"insert_conflicts": 248,
|
"next_pointers_per_node": 1.3316,
|
"memory_used": 19074683,
|
"node_allocs": 87827,
|
"node_frees": 0,
|
"level_node_distribution": {
|
"level0": 66082,
|
"level1": 16211,
|
"level2": 4154,
|
"level3": 1019,
|
"level4": 274,
|
"level5": 69,
|
"level6": 16,
|
"level7": 2,
|
"level8": 0,
|
"level9": 0,
|
"level10": 0,
|
"level11": 0,
|
"level12": 0,
|
"level13": 0,
|
"level14": 0,
|
"level15": 0,
|
"level16": 0,
|
"level17": 0,
|
"level18": 0,
|
"level19": 0,
|
"level20": 0,
|
"level21": 0,
|
"level22": 0,
|
"level23": 0,
|
"level24": 0,
|
"level25": 0,
|
"level26": 0,
|
"level27": 0,
|
"level28": 0,
|
"level29": 0,
|
"level30": 0,
|
"level31": 0,
|
"level32": 0
|
}
|
},
|
"BackStore_0": {
|
"FastHTCount": 21984,
|
"SlowHTCount": 0,
|
"Conflicts": 0,
|
"MemoryInUse": 923328
|
},
|
"BackStore_1": {
|
"FastHTCount": 21934,
|
"SlowHTCount": 0,
|
"Conflicts": 0,
|
"MemoryInUse": 921228
|
},
|
"BackStore_2": {
|
"FastHTCount": 21938,
|
"SlowHTCount": 0,
|
"Conflicts": 0,
|
"MemoryInUse": 921396
|
},
|
"BackStore_3": {
|
"FastHTCount": 21971,
|
"SlowHTCount": 0,
|
"Conflicts": 0,
|
"MemoryInUse": 922782
|
},
|
"data_size": 19074747,
|
"items_count": 87827
|
}
|
}
|
]
|
|