Uploaded image for project: 'Couchbase Server'
  1. Couchbase Server
  2. MB-44324

1 Timer callback failed with LCB_ERR_BUCKET_NOT_FOUND

    XMLWordPrintable

Details

    • Untriaged
    • 1
    • Unknown

    Description

      Build: 7.0.0-4454

      Test: Rebalance multiple kv nodes

      ./testrunner -i /tmp/testexec.120861.ini -p get-cbcollect-info=True,GROUP=bucket_op,skip_log_scan=False,bucket_storage=couchstore -t eventing.eventing_rebalance_collection.EventingRebalanceCollection.test_eventing_rebalance_with_multiple_kv_nodes,doc-per-day=5,dataset=default,nodes_init=5,services_init=kv-kv-kv-eventing-index:n1ql,groups=simple,reset_services=True,GROUP=bucket_op,doc-per-day=20,java_sdk_client=True
       
      Exception: duplicated data in destination bucket which is not expected. Current : 1 Expected : 0  dcp_backlog : 0  TIMERS_IN_PAST : 0 lcb_exceptions : {'172.23.122.117': {}} 

      Seeing Timer callback failed on 172.23.122.117

      {
              "dcp_feed_boundary": "everything",
              "event_processing_stats": {
                  "adhoc_timer_response_received": 12,
                  "agg_messages_sent_to_worker": 109370,
                  "agg_queue_memory": 0,
                  "agg_queue_memory_cap": 53477376,
                  "agg_queue_size": 0,
                  "agg_queue_size_cap": 100000,
                  "agg_timer_feedback_queue_cap": 500,
                  "dcp_deletion": 97933,
                  "dcp_deletion_sent_to_worker": 41338,
                  "dcp_mutation": 221975,
                  "dcp_mutation_sent_to_worker": 58596,
                  "dcp_snapshot": 183630,
                  "dcp_stream_req_counter": 8318,
                  "dcp_stream_req_err_counter": 5304,
                  "dcp_streamend": 1460,
                  "dcp_streamreq": 3014,
                  "dcp_system_event": 10240,
                  "execution_stats": 1096,
                  "failure_stats": 1096,
                  "latency_stats": 1096,
                  "lcb_exception_stats": 1096,
                  "log_level": 1,
                  "num_processed_events": 99934,
                  "processed_events_size": 63702196,
                  "thr_count": 1,
                  "thr_map": 1,
                  "timer_events": 80640,
                  "timer_responses_received": 80640,
                  "v8_init": 1,
                  "v8_load": 1
              },
              "events_remaining": {
                  "dcp_backlog": 0
              },
              "execution_stats": {
                  "agg_queue_memory": 0,
                  "agg_queue_size": 0,
                  "curl": {
                      "delete": 0,
                      "get": 0,
                      "head": 0,
                      "post": 0,
                      "put": 0
                  },
                  "curl_success_count": 0,
                  "dcp_delete_msg_counter": 41338,
                  "dcp_delete_parse_failure": 0,
                  "dcp_mutation_msg_counter": 58596,
                  "dcp_mutation_parse_failure": 0,
                  "enqueued_dcp_delete_msg_counter": 41338,
                  "enqueued_dcp_mutation_msg_counter": 58596,
                  "enqueued_timer_msg_counter": 0,
                  "feedback_queue_size": 0,
                  "filtered_dcp_delete_counter": 0,
                  "filtered_dcp_mutation_counter": 19294,
                  "lcb_retry_failure": 0,
                  "messages_parsed": 109361,
                  "num_processed_events": 99934,
                  "on_delete_failure": 0,
                  "on_delete_success": 40320,
                  "on_update_failure": 0,
                  "on_update_success": 40320,
                  "processed_events_size": 63702196,
                  "timer_callback_failure": 1,
                  "timer_callback_success": 80639,
                  "timer_cancel_counter": 0,
                  "timer_create_counter": 80640,
                  "timer_create_failure": 0,
                  "timer_msg_counter": 80640,
                  "timer_responses_sent": 0,
                  "timestamp": {
                      "84204": "2021-02-13T06:57:27Z"
                  },
                  "uv_msg_parse_failure": 0,
                  "uv_try_write_failure_counter": 0
              },
              "failure_stats": {
                  "app_worker_setting_events_lost": 0,
                  "bkt_ops_cas_mismatch_count": 0,
                  "bucket_op_exception_count": 1,
                  "checkpoint_failure_count": 0,
                  "curl_failure_count": 0,
                  "curl_max_resp_size_exceeded": 0,
                  "curl_non_200_response": 0,
                  "curl_timeout_count": 0,
                  "dcp_events_lost": 0,
                  "debugger_events_lost": 0,
                  "delete_events_lost": 0,
                  "mutation_events_lost": 0,
                  "n1ql_op_exception_count": 0,
                  "timeout_count": 0,
                  "timer_callback_missing_counter": 0,
                  "timer_context_size_exceeded_counter": 0,
                  "timer_events_lost": 0,
                  "timestamp": {
                      "84204": "2021-02-13T06:57:27Z"
                  },
                  "v8worker_events_lost": 0
              },
              "function_id": 1452344633,
              "function_name": "timers",
              "gocb_creds_request_counter": 133,
              "internal_vb_distribution_stats": {
                  "worker_timers_0": "[0-1023]"
              },
              "latency_percentile_stats": {
                  "100": 511900,
                  "50": 1300,
                  "80": 1800,
                  "90": 2400,
                  "95": 3500,
                  "99": 9800
              },
              "lcb_creds_request_counter": 80787,
              "lcb_exception_stats": {
                  "210": 1
              },
              "metastore_stats": {
                  "metastore_delete_err": 0,
                  "metastore_deletes": 0,
                  "metastore_not_found": 0,
                  "metastore_scan": 0,
                  "metastore_scan_due": 0,
                  "metastore_scan_err": 0,
                  "metastore_set": 0,
                  "metastore_set_err": 0
              } 

      Attachments

        1. cbcollect_info_ns_1@172.23.106.67_20210311-185048,ns_server.eventing.log
          25.49 MB
          Sergey Avseyev
        2. test_logs.txt
          818 kB
          Vikas Chaudhary
        3. test.log
          269 kB
          Vikas Chaudhary
        4. test-1.log
          358 kB
          Vikas Chaudhary

        Issue Links

          Activity

            People

              vikas.chaudhary Vikas Chaudhary
              vikas.chaudhary Vikas Chaudhary
              Votes:
              0 Vote for this issue
              Watchers:
              15 Start watching this issue

              Dates

                Created:
                Updated:
                Resolved:

                PagerDuty