Details
-
Bug
-
Resolution: Fixed
-
Critical
-
Cheshire-Cat
-
Untriaged
-
1
-
No
Description
build: 7.0.0-4454
Test
./testrunner -i /tmp/testexec.120861.ini -p get-cbcollect-info=True,GROUP=bucket_op,skip_log_scan=False,bucket_storage=couchstore -t eventing.eventing_rebalance_collection.EventingRebalanceCollection.test_kv_rebalance_in_when_existing_eventing_node_is_processing_mutations,nodes_init=6,services_init=kv-kv-eventing-eventing-eventing-index:n1ql,dataset=default,groups=simple,reset_services=True,doc-per-day=20,GROUP=bucket_op;bucket_op_windows,java_sdk_client=True
|
|
|
Exception: missing data in destination bucket. Current : 40318 Expected : 40320 dcp_backlog : 0 TIMERS_IN_PAST : 0 lcb_exceptions : {'172.23.122.117': {}, '172.23.122.90': {}, '172.23.122.97': {}}
|
Stats
{
|
"dcp_feed_boundary": "everything",
|
"event_processing_stats": {
|
"adhoc_timer_response_received": 2,
|
"agg_messages_sent_to_worker": 17766,
|
"agg_queue_memory": 0,
|
"agg_queue_memory_cap": 20971520,
|
"agg_queue_size": 0,
|
"agg_queue_size_cap": 100000,
|
"agg_timer_feedback_queue_cap": 500,
|
"dcp_mutation": 54088,
|
"dcp_mutation_sent_to_worker": 13432,
|
"dcp_snapshot": 40540,
|
"dcp_stream_req_counter": 1021,
|
"dcp_stream_req_err_counter": 453,
|
"dcp_streamend": 170,
|
"dcp_streamreq": 568,
|
"dcp_system_event": 3410,
|
"execution_stats": 729,
|
"failure_stats": 729,
|
"latency_stats": 729,
|
"lcb_exception_stats": 729,
|
"log_level": 1,
|
"num_processed_events": 13432,
|
"processed_events_size": 12023960,
|
"thr_count": 1,
|
"thr_map": 1,
|
"v8_init": 1,
|
"v8_load": 1
|
},
|
"events_remaining": {
|
"dcp_backlog": 0
|
},
|
"execution_stats": {
|
"agg_queue_memory": 0,
|
"agg_queue_size": 0,
|
"curl": {
|
"delete": 0,
|
"get": 0,
|
"head": 0,
|
"post": 0,
|
"put": 0
|
},
|
"curl_success_count": 0,
|
"dcp_delete_msg_counter": 0,
|
"dcp_delete_parse_failure": 0,
|
"dcp_mutation_msg_counter": 13432,
|
"dcp_mutation_parse_failure": 0,
|
"enqueued_dcp_delete_msg_counter": 0,
|
"enqueued_dcp_mutation_msg_counter": 13432,
|
"enqueued_timer_msg_counter": 0,
|
"feedback_queue_size": 0,
|
"filtered_dcp_delete_counter": 0,
|
"filtered_dcp_mutation_counter": 0,
|
"lcb_retry_failure": 0,
|
"messages_parsed": 17762,
|
"num_processed_events": 13432,
|
"on_delete_failure": 0,
|
"on_delete_success": 0,
|
"on_update_failure": 1,
|
"on_update_success": 13431,
|
"processed_events_size": 12023960,
|
"timer_callback_failure": 0,
|
"timer_callback_success": 0,
|
"timer_cancel_counter": 0,
|
"timer_create_counter": 0,
|
"timer_create_failure": 0,
|
"timer_msg_counter": 0,
|
"timer_responses_sent": 0,
|
"timestamp": {
|
"47458": "2021-02-13T05:24:38Z"
|
},
|
"uv_msg_parse_failure": 0,
|
"uv_try_write_failure_counter": 0
|
},
|
"failure_stats": {
|
"app_worker_setting_events_lost": 0,
|
"bkt_ops_cas_mismatch_count": 0,
|
"bucket_op_exception_count": 0,
|
"checkpoint_failure_count": 0,
|
"curl_failure_count": 0,
|
"curl_max_resp_size_exceeded": 0,
|
"curl_non_200_response": 0,
|
"curl_timeout_count": 0,
|
"dcp_events_lost": 0,
|
"debugger_events_lost": 0,
|
"delete_events_lost": 0,
|
"mutation_events_lost": 0,
|
"n1ql_op_exception_count": 1,
|
"timeout_count": 0,
|
"timer_callback_missing_counter": 0,
|
"timer_context_size_exceeded_counter": 0,
|
"timer_events_lost": 0,
|
"timestamp": {
|
"47458": "2021-02-13T05:24:38Z"
|
},
|
"v8worker_events_lost": 0
|
},
|
"function_id": 1890800864,
|
"function_name": "n1ql",
|
"gocb_creds_request_counter": 58,
|
"internal_vb_distribution_stats": {
|
"worker_n1ql_0": "[342-682]"
|
},
|
"latency_percentile_stats": {
|
"100": 142000,
|
"50": 3400,
|
"80": 4700,
|
"90": 5800,
|
"95": 7100,
|
"99": 11800
|
},
|
"lcb_creds_request_counter": 26931,
|
"lcb_exception_stats": {
|
"12009": 1,
|
"209": 1
|
},
|
"metastore_stats": {},
|
"planner_stats": [
|
{
|
"host_name": "172.23.122.117:8096",
|
"start_vb": 0,
|
"vb_count": 342
|
},
|
{
|
"host_name": "172.23.122.90:8096",
|
"start_vb": 342,
|
"vb_count": 341
|
},
|
{
|
"host_name": "172.23.122.97:8096",
|
"start_vb": 683,
|
"vb_count": 341
|
}
|
],
|
"vb_distribution_stats_from_metadata": {
|
"172.23.122.117:8096": {
|
"worker_n1ql_0": "[0-341]"
|
},
|
"172.23.122.90:8096": {
|
"worker_n1ql_0": "[342-682]"
|
},
|
"172.23.122.97:8096": {
|
"worker_n1ql_0": "[683-1023]"
|
}
|
},
|
"worker_pids": {
|
"worker_n1ql_0": 47458
|
}
|
}
|
Other tests with similar failures
./testrunner -i /tmp/testexec.120861.ini -p get-cbcollect-info=True,GROUP=bucket_op,skip_log_scan=False,bucket_storage=couchstore -t eventing.eventing_rebalance_collection.EventingRebalanceCollection.test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutations,nodes_init=7,services_init=kv-kv-kv-eventing-eventing-eventing-index:n1ql,dataset=default,groups=simple,reset_services=True,doc-per-day=20,GROUP=bucket_op;bucket_op_windows,java_sdk_client=True
|
|
|
Exception: missing data in destination bucket. Current : 40311 Expected : 40320 dcp_backlog : 0 TIMERS_IN_PAST : 0 lcb_exceptions : {'172.23.122.117': {}, '172.23.122.82': {}, '172.23.122.97': {}}
|
{
|
"dcp_feed_boundary": "everything",
|
"event_processing_stats": {
|
"adhoc_timer_response_received": 3,
|
"agg_messages_sent_to_worker": 18480,
|
"agg_queue_memory": 0,
|
"agg_queue_memory_cap": 20971520,
|
"agg_queue_size": 0,
|
"agg_queue_size_cap": 100000,
|
"agg_timer_feedback_queue_cap": 500,
|
"dcp_mutation": 66943,
|
"dcp_mutation_sent_to_worker": 13389,
|
"dcp_snapshot": 40639,
|
"dcp_stream_req_counter": 5805,
|
"dcp_stream_req_err_counter": 4820,
|
"dcp_streamend": 351,
|
"dcp_streamreq": 985,
|
"dcp_system_event": 3410,
|
"execution_stats": 810,
|
"failure_stats": 810,
|
"latency_stats": 810,
|
"lcb_exception_stats": 810,
|
"log_level": 1,
|
"num_processed_events": 13389,
|
"processed_events_size": 11986244,
|
"thr_count": 1,
|
"thr_map": 1,
|
"v8_init": 1,
|
"v8_load": 1
|
},
|
"events_remaining": {
|
"dcp_backlog": 0
|
},
|
"execution_stats": {
|
"agg_queue_memory": 0,
|
"agg_queue_size": 0,
|
"curl": {
|
"delete": 0,
|
"get": 0,
|
"head": 0,
|
"post": 0,
|
"put": 0
|
},
|
"curl_success_count": 0,
|
"dcp_delete_msg_counter": 0,
|
"dcp_delete_parse_failure": 0,
|
"dcp_mutation_msg_counter": 13389,
|
"dcp_mutation_parse_failure": 0,
|
"enqueued_dcp_delete_msg_counter": 0,
|
"enqueued_dcp_mutation_msg_counter": 13389,
|
"enqueued_timer_msg_counter": 0,
|
"feedback_queue_size": 0,
|
"filtered_dcp_delete_counter": 0,
|
"filtered_dcp_mutation_counter": 0,
|
"lcb_retry_failure": 0,
|
"messages_parsed": 18476,
|
"num_processed_events": 13389,
|
"on_delete_failure": 0,
|
"on_delete_success": 0,
|
"on_update_failure": 9,
|
"on_update_success": 13380,
|
"processed_events_size": 11986244,
|
"timer_callback_failure": 0,
|
"timer_callback_success": 0,
|
"timer_cancel_counter": 0,
|
"timer_create_counter": 0,
|
"timer_create_failure": 0,
|
"timer_msg_counter": 0,
|
"timer_responses_sent": 0,
|
"timestamp": {
|
"50026": "2021-02-13T05:49:05Z"
|
},
|
"uv_msg_parse_failure": 0,
|
"uv_try_write_failure_counter": 0
|
},
|
"failure_stats": {
|
"app_worker_setting_events_lost": 0,
|
"bkt_ops_cas_mismatch_count": 0,
|
"bucket_op_exception_count": 0,
|
"checkpoint_failure_count": 0,
|
"curl_failure_count": 0,
|
"curl_max_resp_size_exceeded": 0,
|
"curl_non_200_response": 0,
|
"curl_timeout_count": 0,
|
"dcp_events_lost": 0,
|
"debugger_events_lost": 0,
|
"delete_events_lost": 0,
|
"mutation_events_lost": 0,
|
"n1ql_op_exception_count": 9,
|
"timeout_count": 0,
|
"timer_callback_missing_counter": 0,
|
"timer_context_size_exceeded_counter": 0,
|
"timer_events_lost": 0,
|
"timestamp": {
|
"50026": "2021-02-13T05:49:05Z"
|
},
|
"v8worker_events_lost": 0
|
},
|
"function_id": 1994061678,
|
"function_name": "n1ql",
|
"gocb_creds_request_counter": 88,
|
"internal_vb_distribution_stats": {
|
"worker_n1ql_0": "[683-1023]"
|
},
|
"latency_percentile_stats": {
|
"100": 107900,
|
"50": 3400,
|
"80": 4700,
|
"90": 6000,
|
"95": 7600,
|
"99": 13800
|
},
|
"lcb_creds_request_counter": 26866,
|
"lcb_exception_stats": {
|
"12009": 9,
|
"209": 9
|
},
|
"metastore_stats": {},
|
"planner_stats": [
|
{
|
"host_name": "172.23.122.117:8096",
|
"start_vb": 0,
|
"vb_count": 342
|
},
|
{
|
"host_name": "172.23.122.82:8096",
|
"start_vb": 342,
|
"vb_count": 341
|
},
|
{
|
"host_name": "172.23.122.97:8096",
|
"start_vb": 683,
|
"vb_count": 341
|
}
|
],
|
"vb_distribution_stats_from_metadata": {
|
"172.23.122.117:8096": {
|
"worker_n1ql_0": "[0-341]"
|
},
|
"172.23.122.82:8096": {
|
"worker_n1ql_0": "[342-682]"
|
},
|
"172.23.122.97:8096": {
|
"worker_n1ql_0": "[683-1023]"
|
}
|
},
|
"worker_pids": {
|
"worker_n1ql_0": 50026
|
}
|
}
|
./testrunner -i /tmp/testexec.120861.ini -p get-cbcollect-info=True,GROUP=bucket_op,skip_log_scan=False,bucket_storage=couchstore -t eventing.eventing_rebalance_collection.EventingRebalanceCollection.test_kv_swap_rebalance_when_existing_eventing_node_is_processing_mutations,nodes_init=6,services_init=kv-kv-eventing-eventing-eventing-index:n1ql,dataset=default,groups=simple,reset_services=True,doc-per-day=20,GROUP=bucket_op;bucket_op_windows,java_sdk_client=True
|
|
|
Exception: missing data in destination bucket. Current : 40319 Expected : 40320 dcp_backlog : 0 TIMERS_IN_PAST : 0 lcb_exceptions : {'172.23.122.117': {}, '172.23.122.90': {}, '172.23.122.97': {}}
|
