Critical Description
Test:
./testrunner -i /tmp/win10-bucket-ops.ini -p -t eventing.eventing_upgrade.EventingUpgrade.test_offline_upgrade_with_eventing,nodes_init=4,dataset=default,groups=simple,skip_cleanup=True,initial_version=6.0.0-1693,doc-per-day=1,upgrade_version=6.5.0-4169
|
We deploy bucket op before upgrade , once we upgrade it to mad-hatter its not processing mutation after the upgrade
Waiting for handler code to complete bucket operations... Current : 2016 Expected : 0 ...
|
|
Stats
[
|
{
|
"dcp_feed_boundary": "everything",
|
"event_processing_stats": {
|
"agg_messages_sent_to_worker": 10315,
|
"agg_queue_memory": 0,
|
"agg_queue_memory_cap": 25165824,
|
"agg_queue_size": 0,
|
"agg_queue_size_cap": 300000,
|
"agg_timer_feedback_queue_cap": 1500,
|
"agg_timer_feedback_queue_size": 0,
|
"dcp_stream_req_counter": 1024,
|
"dcp_streamreq": 1024,
|
"execution_stats": 1854,
|
"failure_stats": 1854,
|
"latency_stats": 1854,
|
"lcb_exception_stats": 1854,
|
"log_level": 6,
|
"thr_count": 3,
|
"thr_map": 3,
|
"v8_init": 3,
|
"v8_load": 3
|
},
|
"events_remaining": {
|
"dcp_backlog": 0
|
},
|
"execution_stats": {
|
"agg_queue_memory": 0,
|
"agg_queue_size": 0,
|
"dcp_delete_msg_counter": 0,
|
"dcp_delete_parse_failure": 0,
|
"dcp_mutation_msg_counter": 0,
|
"dcp_mutation_parse_failure": 0,
|
"enqueued_dcp_delete_msg_counter": 0,
|
"enqueued_dcp_mutation_msg_counter": 0,
|
"enqueued_timer_msg_counter": 0,
|
"feedback_queue_size": 0,
|
"filtered_dcp_delete_counter": 0,
|
"filtered_dcp_mutation_counter": 0,
|
"lcb_retry_failure": 0,
|
"messages_parsed": 10273,
|
"on_delete_failure": 0,
|
"on_delete_success": 0,
|
"on_update_failure": 0,
|
"on_update_success": 0,
|
"timer_create_failure": 0,
|
"timer_msg_counter": 0,
|
"timer_responses_sent": 0,
|
"timestamp": {
|
"52029": "2019-09-06T05:13:51Z",
|
"52037": "2019-09-06T05:13:51Z",
|
"52040": "2019-09-06T05:13:51Z"
|
},
|
"uv_try_write_failure_counter": 0
|
},
|
"failure_stats": {
|
"app_worker_setting_events_lost": 0,
|
"bucket_op_exception_count": 0,
|
"checkpoint_failure_count": 0,
|
"dcp_events_lost": 0,
|
"debugger_events_lost": 0,
|
"delete_events_lost": 0,
|
"mutation_events_lost": 0,
|
"n1ql_op_exception_count": 0,
|
"timeout_count": 0,
|
"timer_callback_missing_counter": 0,
|
"timer_context_size_exceeded_counter": 0,
|
"timer_events_lost": 0,
|
"timestamp": {
|
"52029": "2019-09-06T05:13:51Z",
|
"52037": "2019-09-06T05:13:51Z",
|
"52040": "2019-09-06T05:13:51Z"
|
},
|
"v8worker_events_lost": 0
|
},
|
"function_name": "n1ql_op",
|
"gocb_creds_request_counter": 4276,
|
"function_id": 855905484,
|
"internal_vb_distribution_stats": {
|
"worker_n1ql_op_0": "[0-341]",
|
"worker_n1ql_op_1": "[342-682]",
|
"worker_n1ql_op_2": "[683-1023]"
|
},
|
"latency_percentile_stats": {
|
"100": 0,
|
"50": 0,
|
"80": 0,
|
"90": 0,
|
"95": 0,
|
"99": 0
|
},
|
"lcb_creds_request_counter": 36,
|
"lcb_exception_stats": {},
|
"planner_stats": [
|
{
|
"host_name": "172.23.106.83:8096",
|
"start_vb": 0,
|
"vb_count": 1024
|
}
|
],
|
"metastore_stats": {
|
"kvpool_incr": 0,
|
"kvpool_insert": 0,
|
"kvpool_lookup": 1024,
|
"kvpool_remove": 0,
|
"kvpool_replace": 0,
|
"kvpool_upsert": 0
|
},
|
"vb_distribution_stats_from_metadata": {
|
"172.23.106.83:8096": {
|
"worker_n1ql_op_0": "[0-341]",
|
"worker_n1ql_op_1": "[342-682]",
|
"worker_n1ql_op_2": "[683-1023]"
|
}
|
},
|
"worker_pids": {
|
"worker_n1ql_op_0": 52029,
|
"worker_n1ql_op_1": 52037,
|
"worker_n1ql_op_2": 52040
|
}
|
},
|
{
|
"dcp_feed_boundary": "everything",
|
"event_processing_stats": {
|
"agg_messages_sent_to_worker": 9,
|
"agg_queue_memory_cap": 25165824,
|
"agg_queue_size_cap": 300000,
|
"agg_timer_feedback_queue_cap": 1500,
|
"is_bootstrapping": 3,
|
"log_level": 3,
|
"reb_vb_remaining_to_own": 1024,
|
"reb_vb_remaining_to_stream_req": 1024,
|
"thr_count": 3,
|
"thr_map": 3,
|
"worker_spawn_counter": 2117
|
},
|
"events_remaining": {
|
"dcp_backlog": 0
|
},
|
"execution_stats": {
|
"timestamp": {}
|
},
|
"failure_stats": {
|
"timestamp": {}
|
},
|
"function_name": "test_import_function_1",
|
"gocb_creds_request_counter": 4276,
|
"function_id": 0,
|
"internal_vb_distribution_stats": {
|
"worker_test_import_function_1_0": "[]",
|
"worker_test_import_function_1_1": "[]",
|
"worker_test_import_function_1_2": "[]"
|
},
|
"latency_percentile_stats": {
|
"100": 0,
|
"50": 0,
|
"80": 0,
|
"90": 0,
|
"95": 0,
|
"99": 0
|
},
|
"lcb_creds_request_counter": 36,
|
"lcb_exception_stats": {},
|
"planner_stats": [
|
{
|
"host_name": "172.23.106.83:8096",
|
"start_vb": 0,
|
"vb_count": 1024
|
}
|
],
|
"metastore_stats": {
|
"kvpool_incr": 0,
|
"kvpool_insert": 0,
|
"kvpool_lookup": 1024,
|
"kvpool_remove": 0,
|
"kvpool_replace": 0,
|
"kvpool_upsert": 0
|
},
|
"vb_distribution_stats_from_metadata": {
|
"172.23.106.83:8096": {
|
"worker_test_import_function_1_0": "[0-7, 9, 11-341]",
|
"worker_test_import_function_1_1": "[342-347, 349-682]",
|
"worker_test_import_function_1_2": "[684-690, 693-1023]"
|
}
|
},
|
"worker_pids": {
|
"worker_test_import_function_1_0": 83032,
|
"worker_test_import_function_1_1": 83033,
|
"worker_test_import_function_1_2": 83027
|
}
|
},
|
{
|
"dcp_feed_boundary": "everything",
|
"event_processing_stats": {
|
"agg_messages_sent_to_worker": 9010,
|
"agg_queue_memory": 0,
|
"agg_queue_memory_cap": 9437184,
|
"agg_queue_size": 0,
|
"agg_queue_size_cap": 300000,
|
"agg_timer_feedback_queue_cap": 1500,
|
"agg_timer_feedback_queue_size": 0,
|
"dcp_stream_req_counter": 1024,
|
"dcp_streamreq": 1024,
|
"execution_stats": 1593,
|
"failure_stats": 1593,
|
"latency_stats": 1593,
|
"lcb_exception_stats": 1593,
|
"log_level": 6,
|
"thr_count": 3,
|
"thr_map": 3,
|
"v8_init": 3,
|
"v8_load": 3
|
},
|
"events_remaining": {
|
"dcp_backlog": 0
|
},
|
"execution_stats": {
|
"agg_queue_memory": 0,
|
"agg_queue_size": 0,
|
"dcp_delete_msg_counter": 0,
|
"dcp_delete_parse_failure": 0,
|
"dcp_mutation_msg_counter": 0,
|
"dcp_mutation_parse_failure": 0,
|
"enqueued_dcp_delete_msg_counter": 0,
|
"enqueued_dcp_mutation_msg_counter": 0,
|
"enqueued_timer_msg_counter": 0,
|
"feedback_queue_size": 0,
|
"filtered_dcp_delete_counter": 0,
|
"filtered_dcp_mutation_counter": 0,
|
"lcb_retry_failure": 0,
|
"messages_parsed": 8998,
|
"on_delete_failure": 0,
|
"on_delete_success": 0,
|
"on_update_failure": 0,
|
"on_update_success": 0,
|
"timer_create_failure": 0,
|
"timer_msg_counter": 0,
|
"timer_responses_sent": 0,
|
"timestamp": {
|
"52334": "2019-09-06T05:13:52Z",
|
"52344": "2019-09-06T05:13:52Z",
|
"52351": "2019-09-06T05:13:52Z"
|
},
|
"uv_try_write_failure_counter": 0
|
},
|
"failure_stats": {
|
"app_worker_setting_events_lost": 0,
|
"bucket_op_exception_count": 0,
|
"checkpoint_failure_count": 0,
|
"dcp_events_lost": 0,
|
"debugger_events_lost": 0,
|
"delete_events_lost": 0,
|
"mutation_events_lost": 0,
|
"n1ql_op_exception_count": 0,
|
"timeout_count": 0,
|
"timer_callback_missing_counter": 0,
|
"timer_context_size_exceeded_counter": 0,
|
"timer_events_lost": 0,
|
"timestamp": {
|
"52334": "2019-09-06T05:13:52Z",
|
"52344": "2019-09-06T05:13:52Z",
|
"52351": "2019-09-06T05:13:52Z"
|
},
|
"v8worker_events_lost": 0
|
},
|
"function_name": "test_import_function_2",
|
"gocb_creds_request_counter": 4276,
|
"function_id": 3670040965,
|
"internal_vb_distribution_stats": {
|
"worker_test_import_function_2_0": "[0-341]",
|
"worker_test_import_function_2_1": "[342-682]",
|
"worker_test_import_function_2_2": "[683-1023]"
|
},
|
"latency_percentile_stats": {
|
"100": 0,
|
"50": 0,
|
"80": 0,
|
"90": 0,
|
"95": 0,
|
"99": 0
|
},
|
"lcb_creds_request_counter": 36,
|
"lcb_exception_stats": {},
|
"planner_stats": [
|
{
|
"host_name": "172.23.106.83:8096",
|
"start_vb": 0,
|
"vb_count": 1024
|
}
|
],
|
"metastore_stats": {
|
"kvpool_incr": 0,
|
"kvpool_insert": 0,
|
"kvpool_lookup": 1024,
|
"kvpool_remove": 0,
|
"kvpool_replace": 0,
|
"kvpool_upsert": 0,
|
"meta_alarm_missing": 0,
|
"meta_cancel": 0,
|
"meta_cancel_alarm_missing": 0,
|
"meta_cancel_context_missing": 0,
|
"meta_cancel_success": 0,
|
"meta_context_missing": 0,
|
"meta_del": 0,
|
"meta_del_success": 0,
|
"meta_external_span_change": 0,
|
"meta_scan_column": 73100,
|
"meta_scan_column_lookup": 0,
|
"meta_scan_due": 73100,
|
"meta_scan_row": 73100,
|
"meta_scan_row_lookup": 0,
|
"meta_set": 0,
|
"meta_set_success": 0,
|
"meta_span_cas_mismatch": 0,
|
"meta_span_start_change": 0,
|
"meta_span_stop_change": 0,
|
"meta_sync_span": 1024,
|
"meta_timer_in_future_fired": 0,
|
"meta_timer_in_past": 0,
|
"metastore_delete_err": 0,
|
"metastore_deletes": 0,
|
"metastore_not_found": 0,
|
"metastore_scan": 0,
|
"metastore_scan_due": 73100,
|
"metastore_scan_err": 0,
|
"metastore_set": 0,
|
"metastore_set_err": 0
|
},
|
"vb_distribution_stats_from_metadata": {
|
"172.23.106.83:8096": {
|
"worker_test_import_function_2_0": "[0-341]",
|
"worker_test_import_function_2_1": "[342-682]",
|
"worker_test_import_function_2_2": "[683-1023]"
|
}
|
},
|
"worker_pids": {
|
"worker_test_import_function_2_0": 52334,
|
"worker_test_import_function_2_1": 52344,
|
"worker_test_import_function_2_2": 52351
|
}
|
},
|
{
|
"dcp_feed_boundary": "everything",
|
"event_processing_stats": {
|
"agg_messages_sent_to_worker": 10312,
|
"agg_queue_memory": 0,
|
"agg_queue_memory_cap": 25165824,
|
"agg_queue_size": 0,
|
"agg_queue_size_cap": 300000,
|
"agg_timer_feedback_queue_cap": 1500,
|
"agg_timer_feedback_queue_size": 0,
|
"dcp_stream_req_counter": 1024,
|
"dcp_streamreq": 1024,
|
"execution_stats": 1854,
|
"failure_stats": 1854,
|
"latency_stats": 1854,
|
"lcb_exception_stats": 1854,
|
"log_level": 3,
|
"thr_count": 3,
|
"thr_map": 3,
|
"v8_init": 3,
|
"v8_load": 3
|
},
|
"events_remaining": {
|
"dcp_backlog": 0
|
},
|
"execution_stats": {
|
"agg_queue_memory": 0,
|
"agg_queue_size": 0,
|
"dcp_delete_msg_counter": 0,
|
"dcp_delete_parse_failure": 0,
|
"dcp_mutation_msg_counter": 0,
|
"dcp_mutation_parse_failure": 0,
|
"enqueued_dcp_delete_msg_counter": 0,
|
"enqueued_dcp_mutation_msg_counter": 0,
|
"enqueued_timer_msg_counter": 0,
|
"feedback_queue_size": 0,
|
"filtered_dcp_delete_counter": 0,
|
"filtered_dcp_mutation_counter": 0,
|
"lcb_retry_failure": 0,
|
"messages_parsed": 10270,
|
"on_delete_failure": 0,
|
"on_delete_success": 0,
|
"on_update_failure": 0,
|
"on_update_success": 0,
|
"timer_create_failure": 0,
|
"timer_msg_counter": 0,
|
"timer_responses_sent": 0,
|
"timestamp": {
|
"52030": "2019-09-06T05:13:51Z",
|
"52036": "2019-09-06T05:13:51Z",
|
"52044": "2019-09-06T05:13:51Z"
|
},
|
"uv_try_write_failure_counter": 0
|
},
|
"failure_stats": {
|
"app_worker_setting_events_lost": 0,
|
"bucket_op_exception_count": 0,
|
"checkpoint_failure_count": 0,
|
"dcp_events_lost": 0,
|
"debugger_events_lost": 0,
|
"delete_events_lost": 0,
|
"mutation_events_lost": 0,
|
"n1ql_op_exception_count": 0,
|
"timeout_count": 0,
|
"timer_callback_missing_counter": 0,
|
"timer_context_size_exceeded_counter": 0,
|
"timer_events_lost": 0,
|
"timestamp": {
|
"52030": "2019-09-06T05:13:51Z",
|
"52036": "2019-09-06T05:13:51Z",
|
"52044": "2019-09-06T05:13:51Z"
|
},
|
"v8worker_events_lost": 0
|
},
|
"function_name": "bucket_op_curl",
|
"gocb_creds_request_counter": 4276,
|
"function_id": 2073744613,
|
"internal_vb_distribution_stats": {
|
"worker_bucket_op_curl_0": "[0-341]",
|
"worker_bucket_op_curl_1": "[342-682]",
|
"worker_bucket_op_curl_2": "[683-1023]"
|
},
|
"latency_percentile_stats": {
|
"100": 0,
|
"50": 0,
|
"80": 0,
|
"90": 0,
|
"95": 0,
|
"99": 0
|
},
|
"lcb_creds_request_counter": 36,
|
"lcb_exception_stats": {},
|
"planner_stats": [
|
{
|
"host_name": "172.23.106.83:8096",
|
"start_vb": 0,
|
"vb_count": 1024
|
}
|
],
|
"metastore_stats": {
|
"kvpool_incr": 0,
|
"kvpool_insert": 0,
|
"kvpool_lookup": 1024,
|
"kvpool_remove": 0,
|
"kvpool_replace": 0,
|
"kvpool_upsert": 0
|
},
|
"vb_distribution_stats_from_metadata": {
|
"172.23.106.83:8096": {
|
"worker_bucket_op_curl_0": "[0-341]",
|
"worker_bucket_op_curl_1": "[342-682]",
|
"worker_bucket_op_curl_2": "[683-1023]"
|
}
|
},
|
"worker_pids": {
|
"worker_bucket_op_curl_0": 52030,
|
"worker_bucket_op_curl_1": 52036,
|
"worker_bucket_op_curl_2": 52044
|
}
|
},
|
{
|
"dcp_feed_boundary": "everything",
|
"event_processing_stats": {
|
"agg_messages_sent_to_worker": 10315,
|
"agg_queue_memory": 0,
|
"agg_queue_memory_cap": 25165824,
|
"agg_queue_size": 0,
|
"agg_queue_size_cap": 300000,
|
"agg_timer_feedback_queue_cap": 1500,
|
"agg_timer_feedback_queue_size": 0,
|
"dcp_stream_req_counter": 1024,
|
"dcp_streamreq": 1024,
|
"execution_stats": 1854,
|
"failure_stats": 1854,
|
"latency_stats": 1854,
|
"lcb_exception_stats": 1854,
|
"log_level": 6,
|
"thr_count": 3,
|
"thr_map": 3,
|
"v8_init": 3,
|
"v8_load": 3
|
},
|
"events_remaining": {
|
"dcp_backlog": 0
|
},
|
"execution_stats": {
|
"agg_queue_memory": 0,
|
"agg_queue_size": 0,
|
"dcp_delete_msg_counter": 0,
|
"dcp_delete_parse_failure": 0,
|
"dcp_mutation_msg_counter": 0,
|
"dcp_mutation_parse_failure": 0,
|
"enqueued_dcp_delete_msg_counter": 0,
|
"enqueued_dcp_mutation_msg_counter": 0,
|
"enqueued_timer_msg_counter": 0,
|
"feedback_queue_size": 0,
|
"filtered_dcp_delete_counter": 0,
|
"filtered_dcp_mutation_counter": 0,
|
"lcb_retry_failure": 0,
|
"messages_parsed": 10273,
|
"on_delete_failure": 0,
|
"on_delete_success": 0,
|
"on_update_failure": 0,
|
"on_update_success": 0,
|
"timer_create_failure": 0,
|
"timer_msg_counter": 0,
|
"timer_responses_sent": 0,
|
"timestamp": {
|
"52028": "2019-09-06T05:13:51Z",
|
"52032": "2019-09-06T05:13:51Z",
|
"52042": "2019-09-06T05:13:51Z"
|
},
|
"uv_try_write_failure_counter": 0
|
},
|
"failure_stats": {
|
"app_worker_setting_events_lost": 0,
|
"bucket_op_exception_count": 0,
|
"checkpoint_failure_count": 0,
|
"dcp_events_lost": 0,
|
"debugger_events_lost": 0,
|
"delete_events_lost": 0,
|
"mutation_events_lost": 0,
|
"n1ql_op_exception_count": 0,
|
"timeout_count": 0,
|
"timer_callback_missing_counter": 0,
|
"timer_context_size_exceeded_counter": 0,
|
"timer_events_lost": 0,
|
"timestamp": {
|
"52028": "2019-09-06T05:13:51Z",
|
"52032": "2019-09-06T05:13:51Z",
|
"52042": "2019-09-06T05:13:51Z"
|
},
|
"v8worker_events_lost": 0
|
},
|
"function_name": "bucket_op_sbm",
|
"gocb_creds_request_counter": 4276,
|
"function_id": 1456872352,
|
"internal_vb_distribution_stats": {
|
"worker_bucket_op_sbm_0": "[0-341]",
|
"worker_bucket_op_sbm_1": "[342-682]",
|
"worker_bucket_op_sbm_2": "[683-1023]"
|
},
|
"latency_percentile_stats": {
|
"100": 0,
|
"50": 0,
|
"80": 0,
|
"90": 0,
|
"95": 0,
|
"99": 0
|
},
|
"lcb_creds_request_counter": 36,
|
"lcb_exception_stats": {},
|
"planner_stats": [
|
{
|
"host_name": "172.23.106.83:8096",
|
"start_vb": 0,
|
"vb_count": 1024
|
}
|
],
|
"metastore_stats": {
|
"kvpool_incr": 0,
|
"kvpool_insert": 0,
|
"kvpool_lookup": 1024,
|
"kvpool_remove": 0,
|
"kvpool_replace": 0,
|
"kvpool_upsert": 0
|
},
|
"vb_distribution_stats_from_metadata": {
|
"172.23.106.83:8096": {
|
"worker_bucket_op_sbm_0": "[0-341]",
|
"worker_bucket_op_sbm_1": "[342-682]",
|
"worker_bucket_op_sbm_2": "[683-1023]"
|
}
|
},
|
"worker_pids": {
|
"worker_bucket_op_sbm_0": 52028,
|
"worker_bucket_op_sbm_1": 52032,
|
"worker_bucket_op_sbm_2": 52042
|
}
|
}
|
]
|
Attachments
| For Gerrit Dashboard: MB-35842 | ||||||
|---|---|---|---|---|---|---|
| # | Subject | Branch | Project | Status | CR | V |
| 114053,30 | Upgrade changes from older versions to MadHatter | unstable | eventing | Status: MERGED | +2 | +1 |