Uploaded image for project: 'Couchbase Server'
  1. Couchbase Server
  2. MB-45261

Upgrade 6.6.1-> 7.0.0 : Timer creation failing with LCB_ERR_SDK_FEATURE_UNAVAILABLE

    XMLWordPrintable

Details

    • Untriaged
    • 1
    • Unknown

    Description

      Offline Upgrade from 6.6.1-9123 -> 7.0.0-4768

      Test

      ./testrunner.py -i /tmp/win10-bucket-ops.ini -t eventing.eventing_upgrade.EventingUpgrade.test_offline_upgrade_with_eventing,nodes_init=4,dataset=default,groups=simple,upgrade_test=True,initial_version=6.6.1-9213,doc-per-day=10,upgrade_version=7.0.0-4768 

      Seeing following errors

      2021-03-25T02:18:55.615-07:00 [INFO] {"count":20160,"exception":"{\n  \"message\": {\n    \"code\": 0,\n    \"desc\": \"LCB_ERR_SDK_FEATURE_UNAVAILABLE (1029): The requested feature is not supported by the client, either because of settings in the configured instance, or because of options disabled at the time the library was compiled\",\n    \"name\": \"LCB_ERR_SDK_FEATURE_UNAVAILABLE\"\n  },\n  \"stack\": \"Error\\n    at OnUpdate (test_import_function_2.js:7:5)\"\n}","file":"test_import_function_2.js","line":6,"since":"2021-03-25T02:18:55","srcLine":"createTimer(NDtimerCallback,  expiry, meta.id, context);","stack":"Error\n    at OnUpdate (test_import_function_2.js:7:5)”}
      
      

      Stats

      "timer_create_failure": 20160 

      full stats

      [
          {
              "dcp_feed_boundary": "everything",
              "event_processing_stats": {
                  "adhoc_timer_response_received": 3,
                  "agg_messages_sent_to_worker": 27535,
                  "agg_queue_memory": 0,
                  "agg_queue_memory_cap": 33554432,
                  "agg_queue_size": 0,
                  "agg_queue_size_cap": 100000,
                  "agg_timer_feedback_queue_cap": 500,
                  "dcp_mutation": 40320,
                  "dcp_mutation_sent_to_worker": 20160,
                  "dcp_mutation_suppressed_counter": 20160,
                  "dcp_snapshot": 19908,
                  "dcp_stream_req_counter": 1024,
                  "dcp_streamreq": 1024,
                  "execution_stats": 1268,
                  "failure_stats": 1268,
                  "latency_stats": 1268,
                  "lcb_exception_stats": 1268,
                  "log_level": 1,
                  "num_processed_events": 20160,
                  "processed_events_size": 14516812,
                  "thr_count": 1,
                  "thr_map": 1,
                  "v8_init": 1,
                  "v8_load": 1
              },
              "events_remaining": {
                  "dcp_backlog": 0
              },
              "execution_stats": {
                  "agg_queue_memory": 0,
                  "agg_queue_size": 0,
                  "curl": {
                      "delete": 0,
                      "get": 0,
                      "head": 0,
                      "post": 0,
                      "put": 0
                  },
                  "curl_success_count": 0,
                  "dcp_delete_msg_counter": 0,
                  "dcp_delete_parse_failure": 0,
                  "dcp_mutation_msg_counter": 20160,
                  "dcp_mutation_parse_failure": 0,
                  "enqueued_dcp_delete_msg_counter": 0,
                  "enqueued_dcp_mutation_msg_counter": 20160,
                  "enqueued_timer_msg_counter": 0,
                  "feedback_queue_size": 0,
                  "filtered_dcp_delete_counter": 0,
                  "filtered_dcp_mutation_counter": 0,
                  "lcb_retry_failure": 0,
                  "messages_parsed": 27531,
                  "no_op_counter": 0,
                  "num_processed_events": 20160,
                  "on_delete_failure": 0,
                  "on_delete_success": 0,
                  "on_update_failure": 0,
                  "on_update_success": 20160,
                  "processed_events_size": 14516812,
                  "timer_callback_failure": 0,
                  "timer_callback_success": 0,
                  "timer_cancel_counter": 0,
                  "timer_create_counter": 0,
                  "timer_create_failure": 0,
                  "timer_msg_counter": 0,
                  "timer_responses_sent": 0,
                  "timestamp": {
                      "106650": "2021-03-25T09:42:56Z"
                  },
                  "uv_msg_parse_failure": 0,
                  "uv_try_write_failure_counter": 0
              },
              "failure_stats": {
                  "app_worker_setting_events_lost": 0,
                  "bkt_ops_cas_mismatch_count": 0,
                  "bucket_op_exception_count": 0,
                  "checkpoint_failure_count": 0,
                  "curl_failure_count": 0,
                  "curl_max_resp_size_exceeded": 0,
                  "curl_non_200_response": 0,
                  "curl_timeout_count": 0,
                  "dcp_events_lost": 0,
                  "debugger_events_lost": 0,
                  "delete_events_lost": 0,
                  "mutation_events_lost": 0,
                  "n1ql_op_exception_count": 0,
                  "timeout_count": 0,
                  "timer_callback_missing_counter": 0,
                  "timer_context_size_exceeded_counter": 0,
                  "timer_events_lost": 0,
                  "timestamp": {
                      "106650": "2021-03-25T09:42:56Z"
                  },
                  "v8worker_events_lost": 0
              },
              "function_name": "bucket_op_sbm",
              "gocb_creds_request_counter": 1294,
              "function_id": 2169491780,
              "internal_vb_distribution_stats": {
                  "worker_bucket_op_sbm_0": "[0-1023]"
              },
              "latency_percentile_stats": {
                  "50": 300,
                  "80": 400,
                  "90": 600,
                  "95": 700,
                  "99": 1500,
                  "100": 9000
              },
              "lcb_creds_request_counter": 20,
              "lcb_exception_stats": {},
              "planner_stats": [
                  {
                      "host_name": "172.23.106.67:8096",
                      "start_vb": 0,
                      "vb_count": 1024
                  }
              ],
              "metastore_stats": {},
              "vb_distribution_stats_from_metadata": {
                  "172.23.106.67:8096": {
                      "worker_bucket_op_sbm_0": "[0-1023]"
                  }
              },
              "worker_pids": {
                  "worker_bucket_op_sbm_0": 106650
              }
          },
          {
              "dcp_feed_boundary": "everything",
              "event_processing_stats": {
                  "adhoc_timer_response_received": 15,
                  "agg_messages_sent_to_worker": 27309,
                  "agg_queue_memory": 0,
                  "agg_queue_memory_cap": 12582912,
                  "agg_queue_size": 0,
                  "agg_queue_size_cap": 100000,
                  "agg_timer_feedback_queue_cap": 500,
                  "dcp_mutation": 20160,
                  "dcp_mutation_sent_to_worker": 20160,
                  "dcp_snapshot": 1024,
                  "dcp_stream_req_counter": 1024,
                  "dcp_streamreq": 1024,
                  "execution_stats": 1223,
                  "failure_stats": 1223,
                  "latency_stats": 1223,
                  "lcb_exception_stats": 1223,
                  "log_level": 1,
                  "num_processed_events": 20160,
                  "processed_events_size": 14516812,
                  "thr_count": 1,
                  "thr_map": 1,
                  "v8_init": 1,
                  "v8_load": 1
              },
              "events_remaining": {
                  "dcp_backlog": 0
              },
              "execution_stats": {
                  "agg_queue_memory": 0,
                  "agg_queue_size": 0,
                  "curl": {
                      "delete": 0,
                      "get": 20160,
                      "head": 0,
                      "post": 0,
                      "put": 0
                  },
                  "curl_success_count": 20160,
                  "dcp_delete_msg_counter": 0,
                  "dcp_delete_parse_failure": 0,
                  "dcp_mutation_msg_counter": 20160,
                  "dcp_mutation_parse_failure": 0,
                  "enqueued_dcp_delete_msg_counter": 0,
                  "enqueued_dcp_mutation_msg_counter": 20160,
                  "enqueued_timer_msg_counter": 0,
                  "feedback_queue_size": 0,
                  "filtered_dcp_delete_counter": 0,
                  "filtered_dcp_mutation_counter": 0,
                  "lcb_retry_failure": 0,
                  "messages_parsed": 27305,
                  "no_op_counter": 0,
                  "num_processed_events": 20160,
                  "on_delete_failure": 0,
                  "on_delete_success": 0,
                  "on_update_failure": 0,
                  "on_update_success": 20160,
                  "processed_events_size": 14516812,
                  "timer_callback_failure": 0,
                  "timer_callback_success": 0,
                  "timer_cancel_counter": 0,
                  "timer_create_counter": 0,
                  "timer_create_failure": 0,
                  "timer_msg_counter": 0,
                  "timer_responses_sent": 0,
                  "timestamp": {
                      "106719": "2021-03-25T09:42:55Z"
                  },
                  "uv_msg_parse_failure": 0,
                  "uv_try_write_failure_counter": 0
              },
              "failure_stats": {
                  "app_worker_setting_events_lost": 0,
                  "bkt_ops_cas_mismatch_count": 0,
                  "bucket_op_exception_count": 0,
                  "checkpoint_failure_count": 0,
                  "curl_failure_count": 0,
                  "curl_max_resp_size_exceeded": 0,
                  "curl_non_200_response": 0,
                  "curl_timeout_count": 0,
                  "dcp_events_lost": 0,
                  "debugger_events_lost": 0,
                  "delete_events_lost": 0,
                  "mutation_events_lost": 0,
                  "n1ql_op_exception_count": 0,
                  "timeout_count": 0,
                  "timer_callback_missing_counter": 0,
                  "timer_context_size_exceeded_counter": 0,
                  "timer_events_lost": 0,
                  "timestamp": {
                      "106719": "2021-03-25T09:42:55Z"
                  },
                  "v8worker_events_lost": 0
              },
              "function_name": "bucket_op_curl",
              "gocb_creds_request_counter": 1294,
              "function_id": 63159677,
              "internal_vb_distribution_stats": {
                  "worker_bucket_op_curl_0": "[0-1023]"
              },
              "latency_percentile_stats": {
                  "50": 78400,
                  "80": 79500,
                  "90": 80000,
                  "95": 81800,
                  "99": 119300,
                  "100": 2385000
              },
              "lcb_creds_request_counter": 20,
              "lcb_exception_stats": {},
              "planner_stats": [
                  {
                      "host_name": "172.23.106.67:8096",
                      "start_vb": 0,
                      "vb_count": 1024
                  }
              ],
              "metastore_stats": {},
              "vb_distribution_stats_from_metadata": {
                  "172.23.106.67:8096": {
                      "worker_bucket_op_curl_0": "[0-1023]"
                  }
              },
              "worker_pids": {
                  "worker_bucket_op_curl_0": 106719
              }
          },
          {
              "dcp_feed_boundary": "everything",
              "event_processing_stats": {
                  "agg_messages_sent_to_worker": 7879,
                  "agg_queue_memory": 0,
                  "agg_queue_memory_cap": 33554432,
                  "agg_queue_size": 0,
                  "agg_queue_size_cap": 100000,
                  "agg_timer_feedback_queue_cap": 500,
                  "dcp_stream_req_counter": 342,
                  "dcp_streamreq": 342,
                  "execution_stats": 1505,
                  "failure_stats": 1505,
                  "latency_stats": 1505,
                  "lcb_exception_stats": 1505,
                  "log_level": 1,
                  "num_processed_events": 0,
                  "processed_events_size": 0,
                  "reb_vb_remaining_to_own": 682,
                  "reb_vb_remaining_to_stream_req": 682,
                  "thr_count": 1,
                  "thr_map": 1,
                  "v8_init": 1,
                  "v8_load": 1
              },
              "events_remaining": {
                  "dcp_backlog": 0
              },
              "execution_stats": {
                  "agg_queue_memory": 0,
                  "agg_queue_size": 0,
                  "curl": {
                      "delete": 0,
                      "get": 0,
                      "head": 0,
                      "post": 0,
                      "put": 0
                  },
                  "curl_success_count": 0,
                  "dcp_delete_msg_counter": 0,
                  "dcp_delete_parse_failure": 0,
                  "dcp_mutation_msg_counter": 0,
                  "dcp_mutation_parse_failure": 0,
                  "enqueued_dcp_delete_msg_counter": 0,
                  "enqueued_dcp_mutation_msg_counter": 0,
                  "enqueued_timer_msg_counter": 0,
                  "feedback_queue_size": 0,
                  "filtered_dcp_delete_counter": 0,
                  "filtered_dcp_mutation_counter": 0,
                  "lcb_retry_failure": 0,
                  "messages_parsed": 7875,
                  "no_op_counter": 0,
                  "num_processed_events": 0,
                  "on_delete_failure": 0,
                  "on_delete_success": 0,
                  "on_update_failure": 0,
                  "on_update_success": 0,
                  "processed_events_size": 0,
                  "timer_callback_failure": 0,
                  "timer_callback_success": 0,
                  "timer_cancel_counter": 0,
                  "timer_create_counter": 0,
                  "timer_create_failure": 0,
                  "timer_msg_counter": 0,
                  "timer_responses_sent": 0,
                  "timestamp": {
                      "106381": "2021-03-25T09:42:55Z"
                  },
                  "uv_msg_parse_failure": 0,
                  "uv_try_write_failure_counter": 0
              },
              "failure_stats": {
                  "app_worker_setting_events_lost": 0,
                  "bkt_ops_cas_mismatch_count": 0,
                  "bucket_op_exception_count": 0,
                  "checkpoint_failure_count": 0,
                  "curl_failure_count": 0,
                  "curl_max_resp_size_exceeded": 0,
                  "curl_non_200_response": 0,
                  "curl_timeout_count": 0,
                  "dcp_events_lost": 0,
                  "debugger_events_lost": 0,
                  "delete_events_lost": 0,
                  "mutation_events_lost": 0,
                  "n1ql_op_exception_count": 0,
                  "timeout_count": 0,
                  "timer_callback_missing_counter": 0,
                  "timer_context_size_exceeded_counter": 0,
                  "timer_events_lost": 0,
                  "timestamp": {
                      "106381": "2021-03-25T09:42:55Z"
                  },
                  "v8worker_events_lost": 0
              },
              "function_name": "test_import_function_1",
              "gocb_creds_request_counter": 1294,
              "function_id": 3582404440,
              "internal_vb_distribution_stats": {
                  "worker_test_import_function_1_0": "[0-341]"
              },
              "latency_percentile_stats": {
                  "50": 0,
                  "80": 0,
                  "90": 0,
                  "95": 0,
                  "99": 0,
                  "100": 0
              },
              "lcb_creds_request_counter": 20,
              "lcb_exception_stats": {},
              "planner_stats": [
                  {
                      "host_name": "172.23.106.67:8096",
                      "start_vb": 0,
                      "vb_count": 1024
                  }
              ],
              "metastore_stats": {},
              "vb_distribution_stats_from_metadata": {
                  "172.23.106.67:8096": {
                      "worker_test_import_function_1_0": "[0-341]",
                      "worker_test_import_function_1_1": "[342-682]",
                      "worker_test_import_function_1_2": "[683-1023]"
                  }
              },
              "worker_pids": {
                  "worker_test_import_function_1_0": 106381
              }
          },
          {
              "dcp_feed_boundary": "everything",
              "event_processing_stats": {
                  "adhoc_timer_response_received": 2,
                  "agg_messages_sent_to_worker": 29407,
                  "agg_queue_memory": 0,
                  "agg_queue_memory_cap": 33554432,
                  "agg_queue_size": 0,
                  "agg_queue_size_cap": 100000,
                  "agg_timer_feedback_queue_cap": 500,
                  "dcp_mutation": 20160,
                  "dcp_mutation_sent_to_worker": 20160,
                  "dcp_snapshot": 1024,
                  "dcp_stream_req_counter": 1366,
                  "dcp_streamend": 342,
                  "dcp_streamreq": 1366,
                  "execution_stats": 1505,
                  "failure_stats": 1505,
                  "latency_stats": 1505,
                  "lcb_exception_stats": 1505,
                  "log_level": 2,
                  "num_processed_events": 20160,
                  "processed_events_size": 14516812,
                  "thr_count": 1,
                  "thr_map": 1,
                  "v8_init": 1,
                  "v8_load": 1
              },
              "events_remaining": {
                  "dcp_backlog": 0
              },
              "execution_stats": {
                  "agg_queue_memory": 0,
                  "agg_queue_size": 0,
                  "curl": {
                      "delete": 0,
                      "get": 0,
                      "head": 0,
                      "post": 0,
                      "put": 0
                  },
                  "curl_success_count": 0,
                  "dcp_delete_msg_counter": 0,
                  "dcp_delete_parse_failure": 0,
                  "dcp_mutation_msg_counter": 20160,
                  "dcp_mutation_parse_failure": 0,
                  "enqueued_dcp_delete_msg_counter": 0,
                  "enqueued_dcp_mutation_msg_counter": 20160,
                  "enqueued_timer_msg_counter": 0,
                  "feedback_queue_size": 0,
                  "filtered_dcp_delete_counter": 0,
                  "filtered_dcp_mutation_counter": 0,
                  "lcb_retry_failure": 0,
                  "messages_parsed": 29403,
                  "no_op_counter": 0,
                  "num_processed_events": 20160,
                  "on_delete_failure": 0,
                  "on_delete_success": 0,
                  "on_update_failure": 20160,
                  "on_update_success": 0,
                  "processed_events_size": 14516812,
                  "timer_callback_failure": 0,
                  "timer_callback_success": 0,
                  "timer_cancel_counter": 0,
                  "timer_create_counter": 0,
                  "timer_create_failure": 20160,
                  "timer_msg_counter": 0,
                  "timer_responses_sent": 0,
                  "timestamp": {
                      "106372": "2021-03-25T09:42:55Z"
                  },
                  "uv_msg_parse_failure": 0,
                  "uv_try_write_failure_counter": 0
              },
              "failure_stats": {
                  "app_worker_setting_events_lost": 0,
                  "bkt_ops_cas_mismatch_count": 0,
                  "bucket_op_exception_count": 0,
                  "checkpoint_failure_count": 0,
                  "curl_failure_count": 0,
                  "curl_max_resp_size_exceeded": 0,
                  "curl_non_200_response": 0,
                  "curl_timeout_count": 0,
                  "dcp_events_lost": 0,
                  "debugger_events_lost": 0,
                  "delete_events_lost": 0,
                  "mutation_events_lost": 0,
                  "n1ql_op_exception_count": 0,
                  "timeout_count": 0,
                  "timer_callback_missing_counter": 0,
                  "timer_context_size_exceeded_counter": 0,
                  "timer_events_lost": 0,
                  "timestamp": {
                      "106372": "2021-03-25T09:42:55Z"
                  },
                  "v8worker_events_lost": 0
              },
              "function_name": "test_import_function_2",
              "gocb_creds_request_counter": 1294,
              "function_id": 1530978830,
              "internal_vb_distribution_stats": {
                  "worker_test_import_function_2_0": "[0-1023]"
              },
              "latency_percentile_stats": {
                  "50": 100,
                  "80": 100,
                  "90": 100,
                  "95": 200,
                  "99": 900,
                  "100": 11600
              },
              "lcb_creds_request_counter": 20,
              "lcb_exception_stats": {},
              "planner_stats": [
                  {
                      "host_name": "172.23.106.67:8096",
                      "start_vb": 0,
                      "vb_count": 1024
                  }
              ],
              "metastore_stats": {
                  "metastore_delete_err": 0,
                  "metastore_deletes": 0,
                  "metastore_not_found": 0,
                  "metastore_scan": 0,
                  "metastore_scan_due": 0,
                  "metastore_scan_err": 0,
                  "metastore_set": 0,
                  "metastore_set_err": 0
              },
              "vb_distribution_stats_from_metadata": {
                  "172.23.106.67:8096": {
                      "worker_test_import_function_2_0": "[0-1023]"
                  }
              },
              "worker_pids": {
                  "worker_test_import_function_2_0": 106372
              }
          }
      ] 

      Attachments

        Issue Links

          For Gerrit Dashboard: MB-45261
          # Subject Branch Project Status CR V

          Activity

            No work has yet been logged on this issue.

            People

              vikas.chaudhary Vikas Chaudhary
              vikas.chaudhary Vikas Chaudhary
              Votes:
              0 Vote for this issue
              Watchers:
              12 Start watching this issue

              Dates

                Created:
                Updated:
                Resolved:

                Gerrit Reviews

                  There are no open Gerrit changes

                  PagerDuty