Uploaded image for project: 'Couchbase Server'
  1. Couchbase Server
  2. MB-32187

Duplicate mutation when eventing-consumer killed while processing source bucket mutation

    XMLWordPrintable

Details

    • Improvement
    • Resolution: Unresolved
    • Critical
    • feature-backlog
    • 6.5.0
    • eventing
    • None
    • vagrant

    Description

      build:  6.5.0 build 1717

      • create a handler and deploy with source bucket enable

      2018-11-29 22:52:28 | INFO | MainProcess | test_thread | [eventing_base.deploy_function]
      function OnUpdate(doc, meta) {
          log('document', doc);
          try {
              var time_rand = random_gen();
              src_bucket[meta.id + time_rand] = doc;
          } catch(e) {
              //var time_rand = random_gen();
              //dst_bucket[meta.id + time_rand] = doc;
          }
      }
      function random_gen(){
          var rand = Math.floor(Math.random() * 20000000) * Math.floor(Math.random() * 20000000);
          var time_rand = Math.round((new Date()).getTime() / 1000) + rand;
          return time_rand;
      } 

      • Create 2016 doc in source bucket
      • Kill eventing consumer 

      stats

      2018-11-29 23:03:42 | INFO | MainProcess | test_thread | [eventing_base.verify_eventing_results] Stats for Node 10.143.181.102 is
      [
          {
              "dcp_feed_boundary": "everything",
              "event_processing_stats": {
                  "adhoc_timer_response_received": 9,
                  "agg_messages_sent_to_worker": 10267,
                  "agg_queue_memory": 0,
                  "agg_queue_memory_cap": 132120576,
                  "agg_queue_size": 0,
                  "agg_queue_size_cap": 300000,
                  "agg_timer_feedback_queue_cap": 1500,
                  "agg_timer_feedback_queue_size": 0,
                  "dcp_mutation": 6048,
                  "dcp_mutation_sent_to_worker": 2016,
                  "dcp_mutation_suppressed_counter": 4032,
                  "dcp_snapshot": 2053,
                  "dcp_stream_req_counter": 1024,
                  "dcp_streamreq": 1024,
                  "execution_stats": 1803,
                  "failure_stats": 1803,
                  "latency_stats": 1803,
                  "lcb_exception_stats": 1803,
                  "log_level": 3,
                  "thr_count": 3,
                  "thr_map": 3,
                  "v8_init": 3,
                  "v8_load": 3,
                  "worker_spawn_counter": 3
              },
              "events_remaining": {
                  "dcp_backlog": 0
              },
              "execution_stats": {
                  "agg_queue_memory": 0,
                  "agg_queue_size": 0,
                  "dcp_delete_msg_counter": 0,
                  "dcp_mutation_msg_counter": 2016,
                  "enqueued_dcp_delete_msg_counter": 0,
                  "enqueued_dcp_mutation_msg_counter": 2016,
                  "enqueued_timer_msg_counter": 0,
                  "feedback_queue_size": 0,
                  "lcb_retry_failure": 0,
                  "messages_parsed": 10258,
                  "on_delete_failure": 0,
                  "on_delete_success": 0,
                  "on_update_failure": 0,
                  "on_update_success": 2016,
                  "timer_create_failure": 0,
                  "timer_msg_counter": 0,
                  "timer_responses_sent": 0,
                  "timestamp": {
                      "31398": "2018-11-29T17:19:57Z",
                      "31399": "2018-11-29T17:19:57Z",
                      "31400": "2018-11-29T17:19:57Z"
                  },
                  "uv_try_write_failure_counter": 0
              },
              "failure_stats": {
                  "app_worker_setting_events_lost": 0,
                  "bucket_op_exception_count": 0,
                  "checkpoint_failure_count": 0,
                  "dcp_events_lost": 0,
                  "debugger_events_lost": 0,
                  "delete_events_lost": 0,
                  "mutation_events_lost": 0,
                  "n1ql_op_exception_count": 0,
                  "timeout_count": 0,
                  "timer_callback_missing_counter": 0,
                  "timer_context_size_exceeded_counter": 0,
                  "timer_events_lost": 0,
                  "timestamp": {
                      "31398": "2018-11-29T17:19:57Z",
                      "31399": "2018-11-29T17:19:57Z",
                      "31400": "2018-11-29T17:19:57Z"
                  },
                  "v8worker_events_lost": 0
              },
              "function_id": 4032871809,
              "function_name": "Function_349506900_test_killing_eventing_consumer_when_eventing_is_processing_mutations",
              "gocb_creds_request_counter": 14,
              "internal_vb_distribution_stats": {
                  "worker_Function_349506900_test_killing_eventing_consumer_when_eventing_is_processing_mutations_0": "[0-341]",
                  "worker_Function_349506900_test_killing_eventing_consumer_when_eventing_is_processing_mutations_1": "[342-682]",
                  "worker_Function_349506900_test_killing_eventing_consumer_when_eventing_is_processing_mutations_2": "[683-1023]"
              },
              "latency_percentile_stats": {
                  "100": 33000,
                  "50": 1000,
                  "80": 2000,
                  "90": 3000,
                  "95": 5000,
                  "99": 11000
              },
              "lcb_creds_request_counter": 24,
              "lcb_exception_stats": {},
              "metastore_stats": {},
              "planner_stats": [
                  {
                      "host_name": "10.143.181.102:8096",
                      "start_vb": 0,
                      "vb_count": 1024
                  }
              ],
              "vb_distribution_stats_from_metadata": {
                  "10.143.181.102:8096": {
                      "worker_Function_349506900_test_killing_eventing_consumer_when_eventing_is_processing_mutations_0": "[0-341]",
                      "worker_Function_349506900_test_killing_eventing_consumer_when_eventing_is_processing_mutations_1": "[342-682]",
                      "worker_Function_349506900_test_killing_eventing_consumer_when_eventing_is_processing_mutations_2": "[683-1023]"
                  }
              },
              "worker_pids": {
                  "worker_Function_349506900_test_killing_eventing_consumer_when_eventing_is_processing_mutations_0": 31400,
                  "worker_Function_349506900_test_killing_eventing_consumer_when_eventing_is_processing_mutations_1": 31399,
                  "worker_Function_349506900_test_killing_eventing_consumer_when_eventing_is_processing_mutations_2": 31398
              }
          }
      ] 

      Seeing each doc has 2 more doc 

      select meta().id from src_bucket where meta().id like 'query-testemployee10200.7188164%'
       
      [
        {
          "id": "query-testemployee10200.7188164-0"
        },
        {
          "id": "query-testemployee10200.7188164-0175768130249189"
        },
        {
          "id": "query-testemployee10200.7188164-0246392825709960"
        }
      ] 

      Expected : there should be 4032 bucket should be there but saw 6048

      Attachments

        No reviews matched the request. Check your Options in the drop-down menu of this sections header.

        Activity

          People

            srinivasan.raman Srinivasan Raman
            vikas.chaudhary Vikas Chaudhary
            Votes:
            0 Vote for this issue
            Watchers:
            3 Start watching this issue

            Dates

              Created:
              Updated:

              Gerrit Reviews

                There are no open Gerrit changes

                PagerDuty