Details
-
Bug
-
Resolution: Fixed
-
Critical
-
Cheshire-Cat
-
Untriaged
-
1
-
Unknown
Description
Build: 7.0.0-4454
Test: Rebalance multiple kv nodes
./testrunner -i /tmp/testexec.120861.ini -p get-cbcollect-info=True,GROUP=bucket_op,skip_log_scan=False,bucket_storage=couchstore -t eventing.eventing_rebalance_collection.EventingRebalanceCollection.test_eventing_rebalance_with_multiple_kv_nodes,doc-per-day=5,dataset=default,nodes_init=5,services_init=kv-kv-kv-eventing-index:n1ql,groups=simple,reset_services=True,GROUP=bucket_op,doc-per-day=20,java_sdk_client=True
|
|
Exception: duplicated data in destination bucket which is not expected. Current : 1 Expected : 0 dcp_backlog : 0 TIMERS_IN_PAST : 0 lcb_exceptions : {'172.23.122.117': {}}
|
Seeing Timer callback failed on 172.23.122.117
{
|
"dcp_feed_boundary": "everything",
|
"event_processing_stats": {
|
"adhoc_timer_response_received": 12,
|
"agg_messages_sent_to_worker": 109370,
|
"agg_queue_memory": 0,
|
"agg_queue_memory_cap": 53477376,
|
"agg_queue_size": 0,
|
"agg_queue_size_cap": 100000,
|
"agg_timer_feedback_queue_cap": 500,
|
"dcp_deletion": 97933,
|
"dcp_deletion_sent_to_worker": 41338,
|
"dcp_mutation": 221975,
|
"dcp_mutation_sent_to_worker": 58596,
|
"dcp_snapshot": 183630,
|
"dcp_stream_req_counter": 8318,
|
"dcp_stream_req_err_counter": 5304,
|
"dcp_streamend": 1460,
|
"dcp_streamreq": 3014,
|
"dcp_system_event": 10240,
|
"execution_stats": 1096,
|
"failure_stats": 1096,
|
"latency_stats": 1096,
|
"lcb_exception_stats": 1096,
|
"log_level": 1,
|
"num_processed_events": 99934,
|
"processed_events_size": 63702196,
|
"thr_count": 1,
|
"thr_map": 1,
|
"timer_events": 80640,
|
"timer_responses_received": 80640,
|
"v8_init": 1,
|
"v8_load": 1
|
},
|
"events_remaining": {
|
"dcp_backlog": 0
|
},
|
"execution_stats": {
|
"agg_queue_memory": 0,
|
"agg_queue_size": 0,
|
"curl": {
|
"delete": 0,
|
"get": 0,
|
"head": 0,
|
"post": 0,
|
"put": 0
|
},
|
"curl_success_count": 0,
|
"dcp_delete_msg_counter": 41338,
|
"dcp_delete_parse_failure": 0,
|
"dcp_mutation_msg_counter": 58596,
|
"dcp_mutation_parse_failure": 0,
|
"enqueued_dcp_delete_msg_counter": 41338,
|
"enqueued_dcp_mutation_msg_counter": 58596,
|
"enqueued_timer_msg_counter": 0,
|
"feedback_queue_size": 0,
|
"filtered_dcp_delete_counter": 0,
|
"filtered_dcp_mutation_counter": 19294,
|
"lcb_retry_failure": 0,
|
"messages_parsed": 109361,
|
"num_processed_events": 99934,
|
"on_delete_failure": 0,
|
"on_delete_success": 40320,
|
"on_update_failure": 0,
|
"on_update_success": 40320,
|
"processed_events_size": 63702196,
|
"timer_callback_failure": 1,
|
"timer_callback_success": 80639,
|
"timer_cancel_counter": 0,
|
"timer_create_counter": 80640,
|
"timer_create_failure": 0,
|
"timer_msg_counter": 80640,
|
"timer_responses_sent": 0,
|
"timestamp": {
|
"84204": "2021-02-13T06:57:27Z"
|
},
|
"uv_msg_parse_failure": 0,
|
"uv_try_write_failure_counter": 0
|
},
|
"failure_stats": {
|
"app_worker_setting_events_lost": 0,
|
"bkt_ops_cas_mismatch_count": 0,
|
"bucket_op_exception_count": 1,
|
"checkpoint_failure_count": 0,
|
"curl_failure_count": 0,
|
"curl_max_resp_size_exceeded": 0,
|
"curl_non_200_response": 0,
|
"curl_timeout_count": 0,
|
"dcp_events_lost": 0,
|
"debugger_events_lost": 0,
|
"delete_events_lost": 0,
|
"mutation_events_lost": 0,
|
"n1ql_op_exception_count": 0,
|
"timeout_count": 0,
|
"timer_callback_missing_counter": 0,
|
"timer_context_size_exceeded_counter": 0,
|
"timer_events_lost": 0,
|
"timestamp": {
|
"84204": "2021-02-13T06:57:27Z"
|
},
|
"v8worker_events_lost": 0
|
},
|
"function_id": 1452344633,
|
"function_name": "timers",
|
"gocb_creds_request_counter": 133,
|
"internal_vb_distribution_stats": {
|
"worker_timers_0": "[0-1023]"
|
},
|
"latency_percentile_stats": {
|
"100": 511900,
|
"50": 1300,
|
"80": 1800,
|
"90": 2400,
|
"95": 3500,
|
"99": 9800
|
},
|
"lcb_creds_request_counter": 80787,
|
"lcb_exception_stats": {
|
"210": 1
|
},
|
"metastore_stats": {
|
"metastore_delete_err": 0,
|
"metastore_deletes": 0,
|
"metastore_not_found": 0,
|
"metastore_scan": 0,
|
"metastore_scan_due": 0,
|
"metastore_scan_err": 0,
|
"metastore_set": 0,
|
"metastore_set_err": 0
|
}
|