Uploaded image for project: 'Couchbase Server'
  1. Couchbase Server
  2. MB-39739

All mutations failed when producer/consumer killed for cancel timers

    XMLWordPrintable

Details

    • Bug
    • Resolution: Not a Bug
    • Major
    • 6.6.0, 7.0.0
    • 6.6.0
    • qe
    • Ubuntu16

    Description

      Build: 6.6.0 build 7737 

      • Create and deploy the following handler

      function OnUpdate(doc, meta) {
          var expiry = new Date();
          expiry.setSeconds(expiry.getSeconds() + 3000);
          var context = {docID : meta.id, random_text : "e6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh07Aumoe6cZZGHuh07Aumoe6cZZGHuh07Aumoe6"};
          createTimer(timerCallback,  expiry, meta.id, context);
          //log("create timer for:",meta.id)
      }
      function OnDelete(meta) {
          var context = {docID : meta.id};
          try{
          var cancelled=cancelTimer(timerCallback,meta.id);
          log("is timer cancelled:",cancelled," for:",meta.id);
          if(cancelled){
                  try{
                  dst_bucket[meta.id]=meta.id;
                  break;
                  }catch(e){
                      log(e);
                  }
          }
          }catch(e){
              log(e);
          }
      }
       
       
      function NDtimerCallback(context) {
          log("firing delete timer:",context.docID);
          var docID = context.docID;
          while (true) {
          try {
              delete dst_bucket[context.docID];
              break;
          } catch (e) {
              log(e);
              }
          }
      } function timerCallback(context) {
          log("firing update timer:",context.docID);
          var docID = context.docID;
          while (true) {
          try {
              dst_bucket[context.docID]=context.random_text;
              break;
          } catch (e) {
              log(e);
              }
          }
      }

      * load documents in the source bucket

      • Kill consumers 
      • wait for handler to come to the deployed state
      • delete all the documents(this will call cancel timers)
      • kill consumers
      • seeing all the ondelete mutations failed without and error. No error even in logs

      [
          {
              "dcp_feed_boundary": "everything",
              "event_processing_stats": {
                  "adhoc_timer_response_received": 4,
                  "agg_messages_sent_to_worker": 22369,
                  "agg_queue_memory": 0,
                  "agg_queue_memory_cap": 106954752,
                  "agg_queue_size": 0,
                  "agg_queue_size_cap": 300000,
                  "agg_timer_feedback_queue_cap": 1500,
                  "dcp_deletion": 2016,
                  "dcp_deletion_sent_to_worker": 2016,
                  "dcp_snapshot": 877,
                  "dcp_stream_req_counter": 1024,
                  "dcp_streamreq": 1024,
                  "execution_stats": 3861,
                  "failure_stats": 3861,
                  "latency_stats": 3861,
                  "lcb_exception_stats": 3861,
                  "log_level": 3,
                  "num_processed_events": 2016,
                  "processed_events_size": 588728,
                  "thr_count": 3,
                  "thr_map": 3,
                  "v8_init": 3,
                  "v8_load": 3,
                  "worker_spawn_counter": 6
              },
              "events_remaining": {
                  "dcp_backlog": 0
              },
              "execution_stats": {
                  "agg_queue_memory": 0,
                  "agg_queue_size": 0,
                  "curl": {
                      "delete": 0,
                      "get": 0,
                      "head": 0,
                      "post": 0,
                      "put": 0
                  },
                  "dcp_delete_msg_counter": 2016,
                  "dcp_delete_parse_failure": 0,
                  "dcp_mutation_msg_counter": 0,
                  "dcp_mutation_parse_failure": 0,
                  "enqueued_dcp_delete_msg_counter": 2016,
                  "enqueued_dcp_mutation_msg_counter": 0,
                  "enqueued_timer_msg_counter": 0,
                  "feedback_queue_size": 0,
                  "filtered_dcp_delete_counter": 0,
                  "filtered_dcp_mutation_counter": 0,
                  "lcb_retry_failure": 0,
                  "messages_parsed": 22357,
                  "num_processed_events": 2016,
                  "on_delete_failure": 2016,
                  "on_delete_success": 0,
                  "on_update_failure": 0,
                  "on_update_success": 0,
                  "processed_events_size": 588728,
                  "timer_cancel_counter": 0,
                  "timer_create_counter": 0,
                  "timer_create_failure": 0,
                  "timer_msg_counter": 0,
                  "timer_responses_sent": 0,
                  "timestamp": {
                      "6088": "2020-06-04T06:57:42Z",
                      "6094": "2020-06-04T06:57:42Z",
                      "6095": "2020-06-04T06:57:42Z"
                  },
                  "uv_msg_parse_failure": 0,
                  "uv_try_write_failure_counter": 0
              },
              "failure_stats": {
                  "app_worker_setting_events_lost": 0,
                  "bucket_op_exception_count": 0,
                  "checkpoint_failure_count": 0,
                  "curl_non_200_response": 0,
                  "dcp_events_lost": 0,
                  "debugger_events_lost": 0,
                  "delete_events_lost": 0,
                  "mutation_events_lost": 0,
                  "n1ql_op_exception_count": 0,
                  "timeout_count": 0,
                  "timer_callback_missing_counter": 0,
                  "timer_context_size_exceeded_counter": 0,
                  "timer_events_lost": 0,
                  "timestamp": {
                      "6088": "2020-06-04T06:57:42Z",
                      "6094": "2020-06-04T06:57:42Z",
                      "6095": "2020-06-04T06:57:42Z"
                  },
                  "v8worker_events_lost": 0
              },
              "function_name": "Function_445869517_test_killing_eventing_consumer_when_eventing_is_processing_mutations",
              "gocb_creds_request_counter": 20,
              "function_id": 729087126,
              "internal_vb_distribution_stats": {
                  "worker_Function_445869517_test_killing_eventing_consumer_when_eventing_is_processing_mutations_0": "[0-341]",
                  "worker_Function_445869517_test_killing_eventing_consumer_when_eventing_is_processing_mutations_1": "[342-682]",
                  "worker_Function_445869517_test_killing_eventing_consumer_when_eventing_is_processing_mutations_2": "[683-1023]"
              },
              "latency_percentile_stats": {
                  "100": 24600,
                  "50": 2100,
                  "80": 3700,
                  "90": 4900,
                  "95": 6100,
                  "99": 10000
              },
              "lcb_creds_request_counter": 36,
              "lcb_exception_stats": {},
              "planner_stats": [
                  {
                      "host_name": "10.143.192.102:8096",
                      "start_vb": 0,
                      "vb_count": 1024
                  }
              ],
              "metastore_stats": {
                  "metastore_delete_err": 0,
                  "metastore_deletes": 0,
                  "metastore_not_found": 0,
                  "metastore_scan": 0,
                  "metastore_scan_due": 0,
                  "metastore_scan_err": 0,
                  "metastore_set": 0,
                  "metastore_set_err": 0
              },
              "vb_distribution_stats_from_metadata": {
                  "10.143.192.102:8096": {
                      "worker_Function_445869517_test_killing_eventing_consumer_when_eventing_is_processing_mutations_0": "[0-341]",
                      "worker_Function_445869517_test_killing_eventing_consumer_when_eventing_is_processing_mutations_1": "[342-682]",
                      "worker_Function_445869517_test_killing_eventing_consumer_when_eventing_is_processing_mutations_2": "[683-1023]"
                  }
              },
              "worker_pids": {
                  "worker_Function_445869517_test_killing_eventing_consumer_when_eventing_is_processing_mutations_0": 6095,
                  "worker_Function_445869517_test_killing_eventing_consumer_when_eventing_is_processing_mutations_1": 6094,
                  "worker_Function_445869517_test_killing_eventing_consumer_when_eventing_is_processing_mutations_2": 6088
              }
          }
      ] 

      * Similar behaviour if we kill eventing producer

      Attachments

        No reviews matched the request. Check your Options in the drop-down menu of this sections header.

        Activity

          People

            vikas.chaudhary Vikas Chaudhary
            vikas.chaudhary Vikas Chaudhary
            Votes:
            0 Vote for this issue
            Watchers:
            5 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved:

              Gerrit Reviews

                There are no open Gerrit changes

                PagerDuty