on update failure is not zero : Getting 207 with STREAMREQ call failed on dcpFeed

2023-11-07T00:49:56.250-08:00 [Error] DCP[{eventing:b6m0Dez2-16:2772724678_172.23.96.16:11210_worker_bucket-1/scope-1/perf-test1_0_172.23.97.177:8096}] ##0 DcpFeed eventing:b6m0Dez2-17:eventing:b6m0Dez2-16:2772724678_172.23.96.16:11210_worker_bucket-1/scope-1/perf-test1_0_172.23.97.177:8096/0 failed, trying next2023-11-07T00:49:56.250-08:00 [Error] Consumer::dcpRequestStreamHandle [worker_bucket-1/scope-1/perf-test1_0:/tmp/127.0.0.1:8091_0_2772724678.sock:3800636] vb: 0 STREAMREQ call failed on dcpFeed: {eventing:b6m0Dez2-16:2772724678_172.23.96.16:11210_worker_bucket-1/scope-1/perf-test1_0_172.23.97.177:8096}, err: dcp.connection 

throughput100M_get_advbktop_src_bkt_mut_1s_1c_magma_dgm.test 

14:10:20 2023-11-07T00:40:20 [INFO] Stats for 172.23.97.177 : [

14:10:20     {

14:10:20         "dcp_feed_boundary": "from_now",

14:10:20         "event_processing_stats": {

14:10:20             "adhoc_timer_response_received": 584,

14:10:20             "agg_messages_sent_to_worker": 62423279,

14:10:20             "agg_queue_memory": 980593934,

14:10:20             "agg_queue_memory_cap": 5133828096,

14:10:20             "agg_queue_size": 600018,

14:10:20             "agg_queue_size_cap": 2400000,

14:10:20             "agg_timer_feedback_queue_cap": 12000,

14:10:20             "dcp_mutation": 62273743,

14:10:20             "dcp_mutation_sent_to_worker": 62273743,

14:10:20             "dcp_snapshot": 32748156,

14:10:20             "dcp_stream_req_counter": 1024,

14:10:20             "dcp_streamreq": 1024,

14:10:20             "execution_stats": 29664,

14:10:20             "failure_stats": 29664,

14:10:20             "latency_stats": 29664,

14:10:20             "lcb_exception_stats": 29664,

14:10:20             "log_level": 24,

14:10:20             "num_processed_events": 61673737,

14:10:20             "processed_events_size": 86345880324,

14:10:20             "thr_count": 24,

14:10:20             "thr_map": 24,

14:10:20             "v8_init": 24,

14:10:20             "v8_load": 24

14:10:20         },

14:10:20         "events_remaining": {

14:10:20             "dcp_backlog": 1853536

14:10:20         },

14:10:20         "execution_stats": {

14:10:20             "agg_queue_memory": 980593934,

14:10:20             "agg_queue_size": 600018,

14:10:20             "curl": {

14:10:20                 "delete": 0,

14:10:20                 "get": 0,

14:10:20                 "head": 0,

14:10:20                 "post": 0,

14:10:20                 "put": 0

14:10:20             },

14:10:20             "curl_success_count": 0,

14:10:20             "dcp_delete_msg_counter": 0,

14:10:20             "dcp_delete_parse_failure": 0,

14:10:20             "dcp_mutation_msg_counter": 61673749,

14:10:20             "dcp_mutation_parse_failure": 0,

14:10:20             "enqueued_dcp_delete_msg_counter": 0,

14:10:20             "enqueued_dcp_mutation_msg_counter": 62273743,

14:10:20             "enqueued_timer_msg_counter": 0,

14:10:20             "feedback_queue_size": 0,

14:10:20             "filtered_dcp_delete_counter": 0,

14:10:20             "filtered_dcp_mutation_counter": 0,

14:10:20             "lcb_retry_failure": 0,

14:10:20             "messages_parsed": 62423183,

14:10:20             "no_op_counter": 0,

14:10:20             "num_processed_events": 61673737,

14:10:20             "on_delete_failure": 0,

14:10:20             "on_delete_success": 0,

14:10:20             "on_update_failure": 37,

14:10:20             "on_update_success": 61673700,

14:10:20             "processed_events_size": 86345880324,

14:10:20             "timer_callback_failure": 0,

14:10:20             "timer_callback_success": 0,

14:10:20             "timer_cancel_counter": 0,

14:10:20             "timer_create_counter": 0,

14:10:20             "timer_create_failure": 0,

14:10:20             "timer_msg_counter": 0,

14:10:20             "timer_responses_sent": 0,

14:10:20             "timestamp": {

14:10:20                 "3800636": "2023-11-07T08:40:20Z",

14:10:20                 "3800637": "2023-11-07T08:40:20Z",

14:10:20                 "3800664": "2023-11-07T08:40:20Z",

14:10:20                 "3800665": "2023-11-07T08:40:20Z",

14:10:20                 "3800705": "2023-11-07T08:40:20Z",

14:10:20                 "3800754": "2023-11-07T08:40:20Z",

14:10:20                 "3800781": "2023-11-07T08:40:20Z",

14:10:20                 "3800785": "2023-11-07T08:40:20Z",

14:10:20                 "3800813": "2023-11-07T08:40:20Z",

14:10:20                 "3800873": "2023-11-07T08:40:20Z",

14:10:20                 "3800874": "2023-11-07T08:40:20Z",

14:10:20                 "3800907": "2023-11-07T08:40:20Z",

14:10:20                 "3800927": "2023-11-07T08:40:20Z",

14:10:20                 "3800950": "2023-11-07T08:40:20Z",

14:10:20                 "3800974": "2023-11-07T08:40:20Z",

14:10:20                 "3800996": "2023-11-07T08:40:20Z",

14:10:20                 "3801022": "2023-11-07T08:40:20Z",

14:10:20                 "3801046": "2023-11-07T08:40:20Z",

14:10:20                 "3801077": "2023-11-07T08:40:20Z",

14:10:20                 "3801103": "2023-11-07T08:40:20Z",

14:10:20                 "3801155": "2023-11-07T08:40:20Z",

14:10:20                 "3801160": "2023-11-07T08:40:20Z",

14:10:20                 "3801185": "2023-11-07T08:40:20Z",

14:10:20                 "3801216": "2023-11-07T08:40:20Z"

14:10:20             },

14:10:20             "uv_msg_parse_failure": 0,

14:10:20             "uv_try_write_failure_counter": 0

14:10:20         },

14:10:20         "failure_stats": {

14:10:20             "app_worker_setting_events_lost": 0,

14:10:20             "bkt_ops_cas_mismatch_count": 0,

14:10:20             "bucket_cache_overflow_count": 0,

14:10:20             "bucket_op_cache_miss_count": 0,

14:10:20             "bucket_op_exception_count": 37,

14:10:20             "checkpoint_failure_count": 0,

14:10:20             "curl_failure_count": 0,

14:10:20             "curl_max_resp_size_exceeded": 0,

14:10:20             "curl_non_200_response": 0,

14:10:20             "curl_timeout_count": 0,

14:10:20             "dcp_events_lost": 0,

14:10:20             "debugger_events_lost": 0,

14:10:20             "delete_events_lost": 0,

14:10:20             "mutation_events_lost": 0,

14:10:20             "n1ql_op_exception_count": 0,

14:10:20             "timeout_count": 37,

14:10:20             "timer_callback_missing_counter": 0,

14:10:20             "timer_context_size_exceeded_counter": 0,

14:10:20             "timer_events_lost": 0,

14:10:20             "timestamp": {

14:10:20                 "3800636": "2023-11-07T08:40:20Z",

14:10:20                 "3800637": "2023-11-07T08:40:20Z",

14:10:20                 "3800664": "2023-11-07T08:40:20Z",

14:10:20                 "3800665": "2023-11-07T08:40:20Z",

14:10:20                 "3800705": "2023-11-07T08:40:20Z",

14:10:20                 "3800754": "2023-11-07T08:40:20Z",

14:10:20                 "3800781": "2023-11-07T08:40:20Z",

14:10:20                 "3800785": "2023-11-07T08:40:20Z",

14:10:20                 "3800813": "2023-11-07T08:40:20Z",

14:10:20                 "3800873": "2023-11-07T08:40:20Z",

14:10:20                 "3800874": "2023-11-07T08:40:20Z",

14:10:20                 "3800907": "2023-11-07T08:40:20Z",

14:10:20                 "3800927": "2023-11-07T08:40:20Z",

14:10:20                 "3800950": "2023-11-07T08:40:20Z",

14:10:20                 "3800974": "2023-11-07T08:40:20Z",

14:10:20                 "3800996": "2023-11-07T08:40:20Z",

14:10:20                 "3801022": "2023-11-07T08:40:20Z",

14:10:20                 "3801046": "2023-11-07T08:40:20Z",

14:10:20                 "3801077": "2023-11-07T08:40:20Z",

14:10:20                 "3801103": "2023-11-07T08:40:20Z",

14:10:20                 "3801155": "2023-11-07T08:40:20Z",

14:10:20                 "3801160": "2023-11-07T08:40:20Z",

14:10:20                 "3801185": "2023-11-07T08:40:20Z",

14:10:20                 "3801216": "2023-11-07T08:40:20Z"

14:10:20             },

14:10:20             "v8worker_events_lost": 0

14:10:20         },

14:10:20         "function_id": 2772724678,

14:10:20         "function_name": "perf-test1",

14:10:20         "function_scope": {

14:10:20             "bucket": "bucket-1",

14:10:20             "scope": "scope-1"

14:10:20         },

14:10:20         "gocb_creds_request_counter": 12,

14:10:20         "internal_vb_distribution_stats": {

14:10:20             "worker_bucket-1/scope-1/perf-test1_0": "[0-42]",

14:10:20             "worker_bucket-1/scope-1/perf-test1_1": "[43-85]",

14:10:20             "worker_bucket-1/scope-1/perf-test1_10": "[430-472]",

14:10:20             "worker_bucket-1/scope-1/perf-test1_11": "[473-515]",

14:10:20             "worker_bucket-1/scope-1/perf-test1_12": "[516-558]",

14:10:20             "worker_bucket-1/scope-1/perf-test1_13": "[559-601]",

14:10:20             "worker_bucket-1/scope-1/perf-test1_14": "[602-644]",

14:10:20             "worker_bucket-1/scope-1/perf-test1_15": "[645-687]",

14:10:20             "worker_bucket-1/scope-1/perf-test1_16": "[688-729]",

14:10:20             "worker_bucket-1/scope-1/perf-test1_17": "[730-771]",

14:10:20             "worker_bucket-1/scope-1/perf-test1_18": "[772-813]",

14:10:20             "worker_bucket-1/scope-1/perf-test1_19": "[814-855]",

14:10:20             "worker_bucket-1/scope-1/perf-test1_2": "[86-128]",

14:10:20             "worker_bucket-1/scope-1/perf-test1_20": "[856-897]",

14:10:20             "worker_bucket-1/scope-1/perf-test1_21": "[898-939]",

14:10:20             "worker_bucket-1/scope-1/perf-test1_22": "[940-981]",

14:10:20             "worker_bucket-1/scope-1/perf-test1_23": "[982-1023]",

14:10:20             "worker_bucket-1/scope-1/perf-test1_3": "[129-171]",

14:10:20             "worker_bucket-1/scope-1/perf-test1_4": "[172-214]",

14:10:20             "worker_bucket-1/scope-1/perf-test1_5": "[215-257]",

14:10:20             "worker_bucket-1/scope-1/perf-test1_6": "[258-300]",

14:10:20             "worker_bucket-1/scope-1/perf-test1_7": "[301-343]",

14:10:20             "worker_bucket-1/scope-1/perf-test1_8": "[344-386]",

14:10:20             "worker_bucket-1/scope-1/perf-test1_9": "[387-429]"

14:10:20         },

14:10:20         "latency_percentile_stats": {

14:10:20             "100": 9999900,

14:10:20             "50": 300,

14:10:20             "80": 500,

14:10:20             "90": 700,

14:10:20             "95": 800,

14:10:20             "99": 2700

14:10:20         },

14:10:20         "lcb_creds_request_counter": 192,

14:10:20         "lcb_exception_stats": {

14:10:20             "207": 37

14:10:20         },

14:10:20         "metastore_stats": {},

14:10:20         "planner_stats": [

14:10:20             {

14:10:20                 "host_name": "172.23.97.177:8096",

14:10:20                 "start_vb": 0,

14:10:20                 "vb_count": 1024

14:10:20             }

14:10:20         ],

14:10:20         "vb_distribution_stats_from_metadata": {

14:10:20             "172.23.97.177:8096": {

14:10:20                 "worker_bucket-1/scope-1/perf-test1_0": "[0-42]",

14:10:20                 "worker_bucket-1/scope-1/perf-test1_1": "[43-85]",

14:10:20                 "worker_bucket-1/scope-1/perf-test1_10": "[430-472]",

14:10:20                 "worker_bucket-1/scope-1/perf-test1_11": "[473-515]",

14:10:20                 "worker_bucket-1/scope-1/perf-test1_12": "[516-558]",

14:10:20                 "worker_bucket-1/scope-1/perf-test1_13": "[559-601]",

14:10:20                 "worker_bucket-1/scope-1/perf-test1_14": "[602-644]",

14:10:20                 "worker_bucket-1/scope-1/perf-test1_15": "[645-687]",

14:10:20                 "worker_bucket-1/scope-1/perf-test1_16": "[688-729]",

14:10:20                 "worker_bucket-1/scope-1/perf-test1_17": "[730-771]",

14:10:20                 "worker_bucket-1/scope-1/perf-test1_18": "[772-813]",

14:10:20                 "worker_bucket-1/scope-1/perf-test1_19": "[814-855]",

14:10:20                 "worker_bucket-1/scope-1/perf-test1_2": "[86-128]",

14:10:20                 "worker_bucket-1/scope-1/perf-test1_20": "[856-897]",

14:10:20                 "worker_bucket-1/scope-1/perf-test1_21": "[898-939]",

14:10:20                 "worker_bucket-1/scope-1/perf-test1_22": "[940-981]",

14:10:20                 "worker_bucket-1/scope-1/perf-test1_23": "[982-1023]",

14:10:20                 "worker_bucket-1/scope-1/perf-test1_3": "[129-171]",

14:10:20                 "worker_bucket-1/scope-1/perf-test1_4": "[172-214]",

14:10:20                 "worker_bucket-1/scope-1/perf-test1_5": "[215-257]",

14:10:20                 "worker_bucket-1/scope-1/perf-test1_6": "[258-300]",

14:10:20                 "worker_bucket-1/scope-1/perf-test1_7": "[301-343]",

14:10:20                 "worker_bucket-1/scope-1/perf-test1_8": "[344-386]",

14:10:20                 "worker_bucket-1/scope-1/perf-test1_9": "[387-429]"

14:10:20             }

14:10:20         },

14:10:20         "worker_pids": {

14:10:20             "worker_bucket-1/scope-1/perf-test1_0": 3800636,

14:10:20             "worker_bucket-1/scope-1/perf-test1_1": 3800637,

14:10:20             "worker_bucket-1/scope-1/perf-test1_10": 3800874,

14:10:20             "worker_bucket-1/scope-1/perf-test1_11": 3800907,

14:10:20             "worker_bucket-1/scope-1/perf-test1_12": 3800927,

14:10:20             "worker_bucket-1/scope-1/perf-test1_13": 3800950,

14:10:20             "worker_bucket-1/scope-1/perf-test1_14": 3800974,

14:10:20             "worker_bucket-1/scope-1/perf-test1_15": 3800996,

14:10:20             "worker_bucket-1/scope-1/perf-test1_16": 3801022,

14:10:20             "worker_bucket-1/scope-1/perf-test1_17": 3801046,

14:10:20             "worker_bucket-1/scope-1/perf-test1_18": 3801077,

14:10:20             "worker_bucket-1/scope-1/perf-test1_19": 3801103,

14:10:20             "worker_bucket-1/scope-1/perf-test1_2": 3800664,

14:10:20             "worker_bucket-1/scope-1/perf-test1_20": 3801155,

14:10:20             "worker_bucket-1/scope-1/perf-test1_21": 3801160,

14:10:20             "worker_bucket-1/scope-1/perf-test1_22": 3801185,

14:10:20             "worker_bucket-1/scope-1/perf-test1_23": 3801216,

14:10:20             "worker_bucket-1/scope-1/perf-test1_3": 3800665,

14:10:20             "worker_bucket-1/scope-1/perf-test1_4": 3800705,

14:10:20             "worker_bucket-1/scope-1/perf-test1_5": 3800754,

14:10:20             "worker_bucket-1/scope-1/perf-test1_6": 3800781,

14:10:20             "worker_bucket-1/scope-1/perf-test1_7": 3800785,

14:10:20             "worker_bucket-1/scope-1/perf-test1_8": 3800813,

14:10:20             "worker_bucket-1/scope-1/perf-test1_9": 3800873

14:10:20         }

14:10:20     }

14:10:20 ]