Uploaded image for project: 'Couchbase Server'
  1. Couchbase Server
  2. MB-50526

Upgrade: Some bucket operations are failing with LCB_ERR_SSL_CANTVERIFY after online upgrade to neo

    XMLWordPrintable

Details

    • Bug
    • Resolution: Fixed
    • Critical
    • 7.1.0
    • 7.1.0
    • eventing
    • Enterprise Edition 6.6.5 build 10080
      Enterprise Edition 7.1.0 build 2117
    • Untriaged
    • Centos 64-bit
    • 1
    • Unknown

    Description

      STEPS

      1. Cluster consists of 2 nodes (kv + eventing, index + query) running on 6.6.5 - 10080.
      2. Create 4 eventing handlers - bucket_op, timers, curl and n1ql.
      3. Deploy all 4 handlers, load data into source bucket and verify mutations are processed or not.
      4. Disable auto failover, enable n2n encryption and enforce tls.
      5. Pause - resume all 4 handlers.
      6. Load data into source bucket and verify mutations are processed or not.
      7. Perform online upgrade using swap rebalance of the cluster to 7.1.0 - 2017.
      8. Load docs into source bucket and verify mutations are processed or not.

      OBSERVATION
      Few bucket operations fail with the following error.

      2022-01-07T03:23:48.048-08:00 [INFO] {"count":1,"exception":"{\n  \"message\": {\n    \"code\": 8,\n    \"desc\": \"LCB_ERR_SSL_CANTVERIFY (1003): Client could not verify server's certificate\",\n    \"name\": \"LCB_ERR_SSL_CANTVERIFY\"\n  },\n  \"stack\": \"Error\\n    at OnUpdate (bucket_op.js:4:28)\"\n}","file":"bucket_op.js","line":3,"since":"2022-01-07T03:23:02","srcLine":"var result1= couchbase.insert(dst_bucket,meta,doc);","stack":"Error\n    at OnUpdate (bucket_op.js:4:28)"}
       
       
              2022-01-07T03:23:48.028-08:00 [INFO] {"count":1,"exception":"{\n  \"message\": {\n    \"code\": 8,\n    \"desc\": \"LCB_ERR_SSL_CANTVERIFY (1003): Client could not verify server's certificate\",\n    \"name\": \"LCB_ERR_SSL_CANTVERIFY\"\n  },\n  \"stack\": \"Error\\n    at OnUpdate (bucket_op.js:4:28)\"\n}","file":"bucket_op.js","line":3,"since":"2022-01-07T03:22:47","srcLine":"var result1= couchbase.insert(dst_bucket,meta,doc);","stack":"Error\n    at OnUpdate (bucket_op.js:4:28)"}
      

      Eventing Stats

      curl -XGET -k https://Administrator:password@172.23.106.74:18096/api/v1/stats
      [
       {
        "dcp_feed_boundary": "everything",
        "event_processing_stats": {
         "adhoc_timer_response_received": 6,
         "agg_messages_sent_to_worker": 38929,
         "agg_queue_memory": 0,
         "agg_queue_memory_cap": 31457280,
         "agg_queue_size": 0,
         "agg_queue_size_cap": 300000,
         "agg_timer_feedback_queue_cap": 1500,
         "dcp_mutation": 25,
         "dcp_mutation_sent_to_worker": 25,
         "dcp_snapshot": 25,
         "dcp_stream_req_counter": 1539,
         "dcp_stream_req_err_counter": 3,
         "dcp_streamend": 512,
         "dcp_streamreq": 1536,
         "execution_stats": 7464,
         "failure_stats": 7464,
         "latency_stats": 7464,
         "lcb_exception_stats": 7464,
         "log_level": 3,
         "num_processed_events": 25,
         "processed_events_size": 7900,
         "thr_count": 3,
         "thr_map": 3,
         "v8_init": 3,
         "v8_load": 3
        },
        "events_remaining": {
         "dcp_backlog": 0
        },
        "execution_stats": {
         "agg_queue_memory": 0,
         "agg_queue_size": 0,
         "curl": {
          "delete": 0,
          "get": 0,
          "head": 0,
          "post": 0,
          "put": 0
         },
         "curl_success_count": 0,
         "dcp_delete_msg_counter": 0,
         "dcp_delete_parse_failure": 0,
         "dcp_mutation_msg_counter": 25,
         "dcp_mutation_parse_failure": 0,
         "enqueued_dcp_delete_msg_counter": 0,
         "enqueued_dcp_mutation_msg_counter": 25,
         "enqueued_timer_msg_counter": 0,
         "feedback_queue_size": 0,
         "filtered_dcp_delete_counter": 0,
         "filtered_dcp_mutation_counter": 0,
         "lcb_retry_failure": 0,
         "messages_parsed": 38917,
         "no_op_counter": 0,
         "num_processed_events": 25,
         "on_delete_failure": 0,
         "on_delete_success": 0,
         "on_update_failure": 3,
         "on_update_success": 22,
         "processed_events_size": 7900,
         "timer_callback_failure": 0,
         "timer_callback_success": 0,
         "timer_cancel_counter": 0,
         "timer_create_counter": 0,
         "timer_create_failure": 0,
         "timer_msg_counter": 0,
         "timer_responses_sent": 0,
         "timestamp": {
          "51532": "2022-01-21T10:36:20Z",
          "51541": "2022-01-21T10:36:20Z",
          "51544": "2022-01-21T10:36:20Z"
         },
         "uv_msg_parse_failure": 0,
         "uv_try_write_failure_counter": 0
        },
        "failure_stats": {
         "app_worker_setting_events_lost": 0,
         "bkt_ops_cas_mismatch_count": 0,
         "bucket_cache_overflow_count": 0,
         "bucket_op_cache_miss_count": 0,
         "bucket_op_exception_count": 3,
         "checkpoint_failure_count": 0,
         "curl_failure_count": 0,
         "curl_max_resp_size_exceeded": 0,
         "curl_non_200_response": 0,
         "curl_timeout_count": 0,
         "dcp_events_lost": 0,
         "debugger_events_lost": 0,
         "delete_events_lost": 0,
         "mutation_events_lost": 0,
         "n1ql_op_exception_count": 0,
         "timeout_count": 0,
         "timer_callback_missing_counter": 0,
         "timer_context_size_exceeded_counter": 0,
         "timer_events_lost": 0,
         "timestamp": {
          "51532": "2022-01-21T10:36:20Z",
          "51541": "2022-01-21T10:36:20Z",
          "51544": "2022-01-21T10:36:20Z"
         },
         "v8worker_events_lost": 0
        },
        "function_name": "bucket_op",
        "gocb_creds_request_counter": 252,
        "function_id": 2940184271,
        "internal_vb_distribution_stats": {
         "worker_bucket_op_0": "[0-341]",
         "worker_bucket_op_1": "[342-682]",
         "worker_bucket_op_2": "[683-1023]"
        },
        "latency_percentile_stats": {
         "100": 25200,
         "50": 1300,
         "80": 10800,
         "90": 12900,
         "95": 12900,
         "99": 16100
        },
        "lcb_creds_request_counter": 123,
        "lcb_exception_stats": {
         "1003": 3
        },
        "planner_stats": [
         {
          "host_name": "172.23.106.74:18096",
          "start_vb": 0,
          "vb_count": 1024
         }
        ],
        "metastore_stats": {},
        "vb_distribution_stats_from_metadata": {
         "172.23.106.74:18096": {
          "worker_bucket_op_0": "[0-341]",
          "worker_bucket_op_1": "[342-682]",
          "worker_bucket_op_2": "[683-1023]"
         }
        },
        "worker_pids": {
         "worker_bucket_op_0": 51532,
         "worker_bucket_op_1": 51541,
         "worker_bucket_op_2": 51544
        }
       },
       {
        "dcp_feed_boundary": "everything",
        "event_processing_stats": {
         "adhoc_timer_response_received": 4,
         "agg_messages_sent_to_worker": 38686,
         "agg_queue_memory": 0,
         "agg_queue_memory_cap": 31457280,
         "agg_queue_size": 0,
         "agg_queue_size_cap": 300000,
         "agg_timer_feedback_queue_cap": 1500,
         "dcp_mutation": 25,
         "dcp_mutation_sent_to_worker": 25,
         "dcp_snapshot": 25,
         "dcp_stream_req_counter": 1539,
         "dcp_stream_req_err_counter": 3,
         "dcp_streamend": 512,
         "dcp_streamreq": 1536,
         "execution_stats": 7416,
         "failure_stats": 7416,
         "latency_stats": 7416,
         "lcb_exception_stats": 7416,
         "log_level": 3,
         "num_processed_events": 25,
         "processed_events_size": 7900,
         "thr_count": 3,
         "thr_map": 3,
         "v8_init": 3,
         "v8_load": 3
        },
        "events_remaining": {
         "dcp_backlog": 0
        },
        "execution_stats": {
         "agg_queue_memory": 0,
         "agg_queue_size": 0,
         "curl": {
          "delete": 0,
          "get": 25,
          "head": 0,
          "post": 0,
          "put": 0
         },
         "curl_success_count": 25,
         "dcp_delete_msg_counter": 0,
         "dcp_delete_parse_failure": 0,
         "dcp_mutation_msg_counter": 25,
         "dcp_mutation_parse_failure": 0,
         "enqueued_dcp_delete_msg_counter": 0,
         "enqueued_dcp_mutation_msg_counter": 25,
         "enqueued_timer_msg_counter": 0,
         "feedback_queue_size": 0,
         "filtered_dcp_delete_counter": 0,
         "filtered_dcp_mutation_counter": 0,
         "lcb_retry_failure": 0,
         "messages_parsed": 38674,
         "no_op_counter": 0,
         "num_processed_events": 25,
         "on_delete_failure": 0,
         "on_delete_success": 0,
         "on_update_failure": 0,
         "on_update_success": 25,
         "processed_events_size": 7900,
         "timer_callback_failure": 0,
         "timer_callback_success": 0,
         "timer_cancel_counter": 0,
         "timer_create_counter": 0,
         "timer_create_failure": 0,
         "timer_msg_counter": 0,
         "timer_responses_sent": 0,
         "timestamp": {
          "51601": "2022-01-21T10:36:20Z",
          "51606": "2022-01-21T10:36:20Z",
          "51620": "2022-01-21T10:36:20Z"
         },
         "uv_msg_parse_failure": 0,
         "uv_try_write_failure_counter": 0
        },
        "failure_stats": {
         "app_worker_setting_events_lost": 0,
         "bkt_ops_cas_mismatch_count": 0,
         "bucket_cache_overflow_count": 0,
         "bucket_op_cache_miss_count": 0,
         "bucket_op_exception_count": 3,
         "checkpoint_failure_count": 0,
         "curl_failure_count": 0,
         "curl_max_resp_size_exceeded": 0,
         "curl_non_200_response": 25,
         "curl_timeout_count": 0,
         "dcp_events_lost": 0,
         "debugger_events_lost": 0,
         "delete_events_lost": 0,
         "mutation_events_lost": 0,
         "n1ql_op_exception_count": 0,
         "timeout_count": 0,
         "timer_callback_missing_counter": 0,
         "timer_context_size_exceeded_counter": 0,
         "timer_events_lost": 0,
         "timestamp": {
          "51601": "2022-01-21T10:36:20Z",
          "51606": "2022-01-21T10:36:20Z",
          "51620": "2022-01-21T10:36:20Z"
         },
         "v8worker_events_lost": 0
        },
        "function_name": "curl",
        "gocb_creds_request_counter": 252,
        "function_id": 4062727836,
        "internal_vb_distribution_stats": {
         "worker_curl_0": "[0-341]",
         "worker_curl_1": "[342-682]",
         "worker_curl_2": "[683-1023]"
        },
        "latency_percentile_stats": {
         "100": 328600,
         "50": 71900,
         "80": 300500,
         "90": 303200,
         "95": 303300,
         "99": 313000
        },
        "lcb_creds_request_counter": 123,
        "lcb_exception_stats": {
         "1003": 3
        },
        "planner_stats": [
         {
          "host_name": "172.23.106.74:18096",
          "start_vb": 0,
          "vb_count": 1024
         }
        ],
        "metastore_stats": {},
        "vb_distribution_stats_from_metadata": {
         "172.23.106.74:18096": {
          "worker_curl_0": "[0-341]",
          "worker_curl_1": "[342-682]",
          "worker_curl_2": "[683-1023]"
         }
        },
        "worker_pids": {
         "worker_curl_0": 51601,
         "worker_curl_1": 51606,
         "worker_curl_2": 51620
        }
       },
       {
        "dcp_feed_boundary": "everything",
        "event_processing_stats": {
         "adhoc_timer_response_received": 6,
         "agg_messages_sent_to_worker": 38443,
         "agg_queue_memory": 0,
         "agg_queue_memory_cap": 31457280,
         "agg_queue_size": 0,
         "agg_queue_size_cap": 300000,
         "agg_timer_feedback_queue_cap": 1500,
         "dcp_mutation": 25,
         "dcp_mutation_sent_to_worker": 25,
         "dcp_snapshot": 25,
         "dcp_stream_req_counter": 1539,
         "dcp_stream_req_err_counter": 3,
         "dcp_streamend": 512,
         "dcp_streamreq": 1536,
         "execution_stats": 7368,
         "failure_stats": 7368,
         "latency_stats": 7368,
         "lcb_exception_stats": 7368,
         "log_level": 3,
         "num_processed_events": 25,
         "processed_events_size": 7900,
         "thr_count": 3,
         "thr_map": 3,
         "v8_init": 3,
         "v8_load": 3
        },
        "events_remaining": {
         "dcp_backlog": 0
        },
        "execution_stats": {
         "agg_queue_memory": 0,
         "agg_queue_size": 0,
         "curl": {
          "delete": 0,
          "get": 0,
          "head": 0,
          "post": 0,
          "put": 0
         },
         "curl_success_count": 0,
         "dcp_delete_msg_counter": 0,
         "dcp_delete_parse_failure": 0,
         "dcp_mutation_msg_counter": 25,
         "dcp_mutation_parse_failure": 0,
         "enqueued_dcp_delete_msg_counter": 0,
         "enqueued_dcp_mutation_msg_counter": 25,
         "enqueued_timer_msg_counter": 0,
         "feedback_queue_size": 0,
         "filtered_dcp_delete_counter": 0,
         "filtered_dcp_mutation_counter": 0,
         "lcb_retry_failure": 0,
         "messages_parsed": 38431,
         "no_op_counter": 0,
         "num_processed_events": 25,
         "on_delete_failure": 0,
         "on_delete_success": 0,
         "on_update_failure": 0,
         "on_update_success": 25,
         "processed_events_size": 7900,
         "timer_callback_failure": 0,
         "timer_callback_success": 0,
         "timer_cancel_counter": 0,
         "timer_create_counter": 0,
         "timer_create_failure": 0,
         "timer_msg_counter": 0,
         "timer_responses_sent": 0,
         "timestamp": {
          "51658": "2022-01-21T10:36:20Z",
          "51663": "2022-01-21T10:36:20Z",
          "51676": "2022-01-21T10:36:20Z"
         },
         "uv_msg_parse_failure": 0,
         "uv_try_write_failure_counter": 0
        },
        "failure_stats": {
         "app_worker_setting_events_lost": 0,
         "bkt_ops_cas_mismatch_count": 0,
         "bucket_cache_overflow_count": 0,
         "bucket_op_cache_miss_count": 0,
         "bucket_op_exception_count": 0,
         "checkpoint_failure_count": 0,
         "curl_failure_count": 0,
         "curl_max_resp_size_exceeded": 0,
         "curl_non_200_response": 0,
         "curl_timeout_count": 0,
         "dcp_events_lost": 0,
         "debugger_events_lost": 0,
         "delete_events_lost": 0,
         "mutation_events_lost": 0,
         "n1ql_op_exception_count": 0,
         "timeout_count": 0,
         "timer_callback_missing_counter": 0,
         "timer_context_size_exceeded_counter": 0,
         "timer_events_lost": 0,
         "timestamp": {
          "51658": "2022-01-21T10:36:20Z",
          "51663": "2022-01-21T10:36:20Z",
          "51676": "2022-01-21T10:36:20Z"
         },
         "v8worker_events_lost": 0
        },
        "function_name": "n1ql",
        "gocb_creds_request_counter": 252,
        "function_id": 148352305,
        "internal_vb_distribution_stats": {
         "worker_n1ql_0": "[0-341]",
         "worker_n1ql_1": "[342-682]",
         "worker_n1ql_2": "[683-1023]"
        },
        "latency_percentile_stats": {
         "100": 43100,
         "50": 6800,
         "80": 18500,
         "90": 21200,
         "95": 21500,
         "99": 22200
        },
        "lcb_creds_request_counter": 123,
        "lcb_exception_stats": {},
        "planner_stats": [
         {
          "host_name": "172.23.106.74:18096",
          "start_vb": 0,
          "vb_count": 1024
         }
        ],
        "metastore_stats": {},
        "vb_distribution_stats_from_metadata": {
         "172.23.106.74:18096": {
          "worker_n1ql_0": "[0-341]",
          "worker_n1ql_1": "[342-682]",
          "worker_n1ql_2": "[683-1023]"
         }
        },
        "worker_pids": {
         "worker_n1ql_0": 51658,
         "worker_n1ql_1": 51663,
         "worker_n1ql_2": 51676
        }
       },
       {
        "dcp_feed_boundary": "everything",
        "event_processing_stats": {
         "adhoc_timer_response_received": 6,
         "agg_messages_sent_to_worker": 38215,
         "agg_queue_memory": 0,
         "agg_queue_memory_cap": 53477376,
         "agg_queue_size": 0,
         "agg_queue_size_cap": 300000,
         "agg_timer_feedback_queue_cap": 1500,
         "dcp_mutation": 25,
         "dcp_mutation_sent_to_worker": 25,
         "dcp_snapshot": 25,
         "dcp_stream_req_counter": 1539,
         "dcp_stream_req_err_counter": 3,
         "dcp_streamend": 512,
         "dcp_streamreq": 1536,
         "execution_stats": 7323,
         "failure_stats": 7323,
         "latency_stats": 7323,
         "lcb_exception_stats": 7323,
         "log_level": 3,
         "num_processed_events": 25,
         "processed_events_size": 7900,
         "thr_count": 3,
         "thr_map": 3,
         "timer_events": 26,
         "timer_responses_received": 25,
         "v8_init": 3,
         "v8_load": 3
        },
        "events_remaining": {
         "dcp_backlog": 0
        },
        "execution_stats": {
         "agg_queue_memory": 0,
         "agg_queue_size": 0,
         "curl": {
          "delete": 0,
          "get": 0,
          "head": 0,
          "post": 0,
          "put": 0
         },
         "curl_success_count": 0,
         "dcp_delete_msg_counter": 0,
         "dcp_delete_parse_failure": 0,
         "dcp_mutation_msg_counter": 25,
         "dcp_mutation_parse_failure": 0,
         "enqueued_dcp_delete_msg_counter": 0,
         "enqueued_dcp_mutation_msg_counter": 25,
         "enqueued_timer_msg_counter": 0,
         "feedback_queue_size": 0,
         "filtered_dcp_delete_counter": 0,
         "filtered_dcp_mutation_counter": 0,
         "lcb_retry_failure": 0,
         "messages_parsed": 38203,
         "no_op_counter": 0,
         "num_processed_events": 25,
         "on_delete_failure": 0,
         "on_delete_success": 0,
         "on_update_failure": 0,
         "on_update_success": 25,
         "processed_events_size": 7900,
         "timer_callback_failure": 4,
         "timer_callback_success": 22,
         "timer_cancel_counter": 0,
         "timer_create_counter": 25,
         "timer_create_failure": 0,
         "timer_msg_counter": 26,
         "timer_responses_sent": 0,
         "timestamp": {
          "51712": "2022-01-21T10:36:21Z",
          "51715": "2022-01-21T10:36:21Z",
          "51730": "2022-01-21T10:36:21Z"
         },
         "uv_msg_parse_failure": 0,
         "uv_try_write_failure_counter": 0
        },
        "failure_stats": {
         "app_worker_setting_events_lost": 0,
         "bkt_ops_cas_mismatch_count": 0,
         "bucket_cache_overflow_count": 0,
         "bucket_op_cache_miss_count": 0,
         "bucket_op_exception_count": 4,
         "checkpoint_failure_count": 0,
         "curl_failure_count": 0,
         "curl_max_resp_size_exceeded": 0,
         "curl_non_200_response": 0,
         "curl_timeout_count": 0,
         "dcp_events_lost": 0,
         "debugger_events_lost": 0,
         "delete_events_lost": 0,
         "mutation_events_lost": 0,
         "n1ql_op_exception_count": 0,
         "timeout_count": 0,
         "timer_callback_missing_counter": 0,
         "timer_context_size_exceeded_counter": 0,
         "timer_events_lost": 0,
         "timestamp": {
          "51712": "2022-01-21T10:36:21Z",
          "51715": "2022-01-21T10:36:21Z",
          "51730": "2022-01-21T10:36:21Z"
         },
         "v8worker_events_lost": 0
        },
        "function_name": "timers",
        "gocb_creds_request_counter": 252,
        "function_id": 1360836055,
        "internal_vb_distribution_stats": {
         "worker_timers_0": "[0-341]",
         "worker_timers_1": "[342-682]",
         "worker_timers_2": "[683-1023]"
        },
        "latency_percentile_stats": {
         "100": 24500,
         "50": 2000,
         "80": 3600,
         "90": 13000,
         "95": 13800,
         "99": 14600
        },
        "lcb_creds_request_counter": 123,
        "lcb_exception_stats": {
         "1003": 4
        },
        "planner_stats": [
         {
          "host_name": "172.23.106.74:18096",
          "start_vb": 0,
          "vb_count": 1024
         }
        ],
        "metastore_stats": {
         "metastore_delete_err": 0,
         "metastore_deletes": 0,
         "metastore_not_found": 0,
         "metastore_scan": 0,
         "metastore_scan_due": 0,
         "metastore_scan_err": 0,
         "metastore_set": 0,
         "metastore_set_err": 0
        },
        "vb_distribution_stats_from_metadata": {
         "172.23.106.74:18096": {
          "worker_timers_0": "[0-341]",
          "worker_timers_1": "[342-682]",
          "worker_timers_2": "[683-1023]"
         }
        },
        "worker_pids": {
         "worker_timers_0": 51712,
         "worker_timers_1": 51715,
         "worker_timers_2": 51730
        }
       }
      ]
      

      On 172.23.106.74
      ns_server.eventing.log

      2022-01-21T02:00:15.632-08:00 [Error] ServiceMgr::getCreds Failed to get credentials for endpoint: 172.23.106.64:11207, err: Unable to find given hostport in cbauth database: `172.23.106.64:11207'
      2022-01-21T02:00:15.632-08:00 [Error] ServiceMgr::getCreds Failed to get credentials for endpoint: 172.23.106.64:11207, err: Unable to find given hostport in cbauth database: `172.23.106.64:11207'
      2022-01-21T02:00:15.632-08:00 [Info] eventing-consumer [worker_timers_0:/tmp/127.0.0.1:8091_0_1360836055.sock:51712] Unable to get creds: non 200 status in header: 
      2022-01-21T02:00:15.632-08:00 [Info] eventing-consumer [worker_timers_0:/tmp/127.0.0.1:8091_0_1360836055.sock:51712] Unable to get creds: non 200 status in header: 
      2022-01-21T02:00:15.639-08:00 [Info] eventing-consumer [worker_timers_0:/tmp/127.0.0.1:8091_0_1360836055.sock:51712] [lcb,SSL L:162 I:382510552] error:1416F086:SSL routines:tls_process_server_certificate:certificate verify failed
      2022-01-21T02:00:15.639-08:00 [Info] eventing-consumer [worker_timers_0:/tmp/127.0.0.1:8091_0_1360836055.sock:51712] [lcb,server L:968 I:382510552] <NOHOST:NOPORT> (CTX=(nil),,SRV=0x7fea2013c160,IX=0) Connection attempt failed. Received LCB_ERR_SSL_CANTVERIFY (1003) from libcouchbase, received 0 (Success) from operating system
      2022-01-21T02:00:15.639-08:00 [Info] eventing-consumer [worker_timers_0:/tmp/127.0.0.1:8091_0_1360836055.sock:51712] [lcb,server L:825 I:382510552] <NOHOST:NOPORT> (CTX=(nil),,SRV=0x7fea2013c160,IX=0) Failing command (pkt=0x7fea0c001a60, opaque=1444, opcode=0xb5) with error LCB_ERR_SSL_CANTVERIFY (1003)
      2022-01-21T02:00:15.639-08:00 [Info] eventing-consumer [worker_timers_0:/tmp/127.0.0.1:8091_0_1360836055.sock:51712] [lcb,cccp L:187 I:382510552] <NOHOST:NOPORT> (CTX=(nil),) Could not get configuration: LCB_ERR_SSL_CANTVERIFY (1003)
      

      Attachments

        Issue Links

          No reviews matched the request. Check your Options in the drop-down menu of this sections header.

          Activity

            People

              sujay.gad Sujay Gad
              sujay.gad Sujay Gad
              Votes:
              0 Vote for this issue
              Watchers:
              3 Start watching this issue

              Dates

                Created:
                Updated:
                Resolved:

                Gerrit Reviews

                  There are no open Gerrit changes

                  PagerDuty