Uploaded image for project: 'Couchbase Server'
  1. Couchbase Server
  2. MB-44328

Eventing: N1ql queries failing with CAS mismatch (LCB 209)

    XMLWordPrintable

Details

    • Untriaged
    • 1
    • No

    Description

      build: 7.0.0-4454

      Test

       ./testrunner -i /tmp/testexec.120861.ini -p get-cbcollect-info=True,GROUP=bucket_op,skip_log_scan=False,bucket_storage=couchstore -t eventing.eventing_rebalance_collection.EventingRebalanceCollection.test_kv_rebalance_in_when_existing_eventing_node_is_processing_mutations,nodes_init=6,services_init=kv-kv-eventing-eventing-eventing-index:n1ql,dataset=default,groups=simple,reset_services=True,doc-per-day=20,GROUP=bucket_op;bucket_op_windows,java_sdk_client=True
       
      Exception: missing data in destination bucket. Current : 40318 Expected : 40320  dcp_backlog : 0  TIMERS_IN_PAST : 0 lcb_exceptions : {'172.23.122.117': {}, '172.23.122.90': {}, '172.23.122.97': {}}

      Stats

       {
              "dcp_feed_boundary": "everything",
              "event_processing_stats": {
                  "adhoc_timer_response_received": 2,
                  "agg_messages_sent_to_worker": 17766,
                  "agg_queue_memory": 0,
                  "agg_queue_memory_cap": 20971520,
                  "agg_queue_size": 0,
                  "agg_queue_size_cap": 100000,
                  "agg_timer_feedback_queue_cap": 500,
                  "dcp_mutation": 54088,
                  "dcp_mutation_sent_to_worker": 13432,
                  "dcp_snapshot": 40540,
                  "dcp_stream_req_counter": 1021,
                  "dcp_stream_req_err_counter": 453,
                  "dcp_streamend": 170,
                  "dcp_streamreq": 568,
                  "dcp_system_event": 3410,
                  "execution_stats": 729,
                  "failure_stats": 729,
                  "latency_stats": 729,
                  "lcb_exception_stats": 729,
                  "log_level": 1,
                  "num_processed_events": 13432,
                  "processed_events_size": 12023960,
                  "thr_count": 1,
                  "thr_map": 1,
                  "v8_init": 1,
                  "v8_load": 1
              },
              "events_remaining": {
                  "dcp_backlog": 0
              },
              "execution_stats": {
                  "agg_queue_memory": 0,
                  "agg_queue_size": 0,
                  "curl": {
                      "delete": 0,
                      "get": 0,
                      "head": 0,
                      "post": 0,
                      "put": 0
                  },
                  "curl_success_count": 0,
                  "dcp_delete_msg_counter": 0,
                  "dcp_delete_parse_failure": 0,
                  "dcp_mutation_msg_counter": 13432,
                  "dcp_mutation_parse_failure": 0,
                  "enqueued_dcp_delete_msg_counter": 0,
                  "enqueued_dcp_mutation_msg_counter": 13432,
                  "enqueued_timer_msg_counter": 0,
                  "feedback_queue_size": 0,
                  "filtered_dcp_delete_counter": 0,
                  "filtered_dcp_mutation_counter": 0,
                  "lcb_retry_failure": 0,
                  "messages_parsed": 17762,
                  "num_processed_events": 13432,
                  "on_delete_failure": 0,
                  "on_delete_success": 0,
                  "on_update_failure": 1,
                  "on_update_success": 13431,
                  "processed_events_size": 12023960,
                  "timer_callback_failure": 0,
                  "timer_callback_success": 0,
                  "timer_cancel_counter": 0,
                  "timer_create_counter": 0,
                  "timer_create_failure": 0,
                  "timer_msg_counter": 0,
                  "timer_responses_sent": 0,
                  "timestamp": {
                      "47458": "2021-02-13T05:24:38Z"
                  },
                  "uv_msg_parse_failure": 0,
                  "uv_try_write_failure_counter": 0
              },
              "failure_stats": {
                  "app_worker_setting_events_lost": 0,
                  "bkt_ops_cas_mismatch_count": 0,
                  "bucket_op_exception_count": 0,
                  "checkpoint_failure_count": 0,
                  "curl_failure_count": 0,
                  "curl_max_resp_size_exceeded": 0,
                  "curl_non_200_response": 0,
                  "curl_timeout_count": 0,
                  "dcp_events_lost": 0,
                  "debugger_events_lost": 0,
                  "delete_events_lost": 0,
                  "mutation_events_lost": 0,
                  "n1ql_op_exception_count": 1,
                  "timeout_count": 0,
                  "timer_callback_missing_counter": 0,
                  "timer_context_size_exceeded_counter": 0,
                  "timer_events_lost": 0,
                  "timestamp": {
                      "47458": "2021-02-13T05:24:38Z"
                  },
                  "v8worker_events_lost": 0
              },
              "function_id": 1890800864,
              "function_name": "n1ql",
              "gocb_creds_request_counter": 58,
              "internal_vb_distribution_stats": {
                  "worker_n1ql_0": "[342-682]"
              },
              "latency_percentile_stats": {
                  "100": 142000,
                  "50": 3400,
                  "80": 4700,
                  "90": 5800,
                  "95": 7100,
                  "99": 11800
              },
              "lcb_creds_request_counter": 26931,
              "lcb_exception_stats": {
                  "12009": 1,
                  "209": 1
              },
              "metastore_stats": {},
              "planner_stats": [
                  {
                      "host_name": "172.23.122.117:8096",
                      "start_vb": 0,
                      "vb_count": 342
                  },
                  {
                      "host_name": "172.23.122.90:8096",
                      "start_vb": 342,
                      "vb_count": 341
                  },
                  {
                      "host_name": "172.23.122.97:8096",
                      "start_vb": 683,
                      "vb_count": 341
                  }
              ],
              "vb_distribution_stats_from_metadata": {
                  "172.23.122.117:8096": {
                      "worker_n1ql_0": "[0-341]"
                  },
                  "172.23.122.90:8096": {
                      "worker_n1ql_0": "[342-682]"
                  },
                  "172.23.122.97:8096": {
                      "worker_n1ql_0": "[683-1023]"
                  }
              },
              "worker_pids": {
                  "worker_n1ql_0": 47458
              }
          }

      Other tests with similar failures

      ./testrunner -i /tmp/testexec.120861.ini -p get-cbcollect-info=True,GROUP=bucket_op,skip_log_scan=False,bucket_storage=couchstore -t eventing.eventing_rebalance_collection.EventingRebalanceCollection.test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutations,nodes_init=7,services_init=kv-kv-kv-eventing-eventing-eventing-index:n1ql,dataset=default,groups=simple,reset_services=True,doc-per-day=20,GROUP=bucket_op;bucket_op_windows,java_sdk_client=True
       
      Exception: missing data in destination bucket. Current : 40311 Expected : 40320  dcp_backlog : 0  TIMERS_IN_PAST : 0 lcb_exceptions : {'172.23.122.117': {}, '172.23.122.82': {}, '172.23.122.97': {}} 

      {
              "dcp_feed_boundary": "everything",
              "event_processing_stats": {
                  "adhoc_timer_response_received": 3,
                  "agg_messages_sent_to_worker": 18480,
                  "agg_queue_memory": 0,
                  "agg_queue_memory_cap": 20971520,
                  "agg_queue_size": 0,
                  "agg_queue_size_cap": 100000,
                  "agg_timer_feedback_queue_cap": 500,
                  "dcp_mutation": 66943,
                  "dcp_mutation_sent_to_worker": 13389,
                  "dcp_snapshot": 40639,
                  "dcp_stream_req_counter": 5805,
                  "dcp_stream_req_err_counter": 4820,
                  "dcp_streamend": 351,
                  "dcp_streamreq": 985,
                  "dcp_system_event": 3410,
                  "execution_stats": 810,
                  "failure_stats": 810,
                  "latency_stats": 810,
                  "lcb_exception_stats": 810,
                  "log_level": 1,
                  "num_processed_events": 13389,
                  "processed_events_size": 11986244,
                  "thr_count": 1,
                  "thr_map": 1,
                  "v8_init": 1,
                  "v8_load": 1
              },
              "events_remaining": {
                  "dcp_backlog": 0
              },
              "execution_stats": {
                  "agg_queue_memory": 0,
                  "agg_queue_size": 0,
                  "curl": {
                      "delete": 0,
                      "get": 0,
                      "head": 0,
                      "post": 0,
                      "put": 0
                  },
                  "curl_success_count": 0,
                  "dcp_delete_msg_counter": 0,
                  "dcp_delete_parse_failure": 0,
                  "dcp_mutation_msg_counter": 13389,
                  "dcp_mutation_parse_failure": 0,
                  "enqueued_dcp_delete_msg_counter": 0,
                  "enqueued_dcp_mutation_msg_counter": 13389,
                  "enqueued_timer_msg_counter": 0,
                  "feedback_queue_size": 0,
                  "filtered_dcp_delete_counter": 0,
                  "filtered_dcp_mutation_counter": 0,
                  "lcb_retry_failure": 0,
                  "messages_parsed": 18476,
                  "num_processed_events": 13389,
                  "on_delete_failure": 0,
                  "on_delete_success": 0,
                  "on_update_failure": 9,
                  "on_update_success": 13380,
                  "processed_events_size": 11986244,
                  "timer_callback_failure": 0,
                  "timer_callback_success": 0,
                  "timer_cancel_counter": 0,
                  "timer_create_counter": 0,
                  "timer_create_failure": 0,
                  "timer_msg_counter": 0,
                  "timer_responses_sent": 0,
                  "timestamp": {
                      "50026": "2021-02-13T05:49:05Z"
                  },
                  "uv_msg_parse_failure": 0,
                  "uv_try_write_failure_counter": 0
              },
              "failure_stats": {
                  "app_worker_setting_events_lost": 0,
                  "bkt_ops_cas_mismatch_count": 0,
                  "bucket_op_exception_count": 0,
                  "checkpoint_failure_count": 0,
                  "curl_failure_count": 0,
                  "curl_max_resp_size_exceeded": 0,
                  "curl_non_200_response": 0,
                  "curl_timeout_count": 0,
                  "dcp_events_lost": 0,
                  "debugger_events_lost": 0,
                  "delete_events_lost": 0,
                  "mutation_events_lost": 0,
                  "n1ql_op_exception_count": 9,
                  "timeout_count": 0,
                  "timer_callback_missing_counter": 0,
                  "timer_context_size_exceeded_counter": 0,
                  "timer_events_lost": 0,
                  "timestamp": {
                      "50026": "2021-02-13T05:49:05Z"
                  },
                  "v8worker_events_lost": 0
              },
              "function_id": 1994061678,
              "function_name": "n1ql",
              "gocb_creds_request_counter": 88,
              "internal_vb_distribution_stats": {
                  "worker_n1ql_0": "[683-1023]"
              },
              "latency_percentile_stats": {
                  "100": 107900,
                  "50": 3400,
                  "80": 4700,
                  "90": 6000,
                  "95": 7600,
                  "99": 13800
              },
              "lcb_creds_request_counter": 26866,
              "lcb_exception_stats": {
                  "12009": 9,
                  "209": 9
              },
              "metastore_stats": {},
              "planner_stats": [
                  {
                      "host_name": "172.23.122.117:8096",
                      "start_vb": 0,
                      "vb_count": 342
                  },
                  {
                      "host_name": "172.23.122.82:8096",
                      "start_vb": 342,
                      "vb_count": 341
                  },
                  {
                      "host_name": "172.23.122.97:8096",
                      "start_vb": 683,
                      "vb_count": 341
                  }
              ],
              "vb_distribution_stats_from_metadata": {
                  "172.23.122.117:8096": {
                      "worker_n1ql_0": "[0-341]"
                  },
                  "172.23.122.82:8096": {
                      "worker_n1ql_0": "[342-682]"
                  },
                  "172.23.122.97:8096": {
                      "worker_n1ql_0": "[683-1023]"
                  }
              },
              "worker_pids": {
                  "worker_n1ql_0": 50026
              }
          } 

      ./testrunner -i /tmp/testexec.120861.ini -p get-cbcollect-info=True,GROUP=bucket_op,skip_log_scan=False,bucket_storage=couchstore -t eventing.eventing_rebalance_collection.EventingRebalanceCollection.test_kv_swap_rebalance_when_existing_eventing_node_is_processing_mutations,nodes_init=6,services_init=kv-kv-eventing-eventing-eventing-index:n1ql,dataset=default,groups=simple,reset_services=True,doc-per-day=20,GROUP=bucket_op;bucket_op_windows,java_sdk_client=True
       
      Exception: missing data in destination bucket. Current : 40319 Expected : 40320  dcp_backlog : 0  TIMERS_IN_PAST : 0 lcb_exceptions : {'172.23.122.117': {}, '172.23.122.90': {}, '172.23.122.97': {}} 

      Attachments

        1. 172.23.104.97-20210227-1719-diag.zip
          23.83 MB
        2. 172.23.105.238-20210227-1719-diag.zip
          19.47 MB
        3. 172.23.105.240-20210227-1719-diag.zip
          20.54 MB
        4. 172.23.105.247-20210227-1719-diag.zip
          22.99 MB
        5. 172.23.105.249-20210227-1719-diag.zip
          16.33 MB
        6. 172.23.106.1-20210227-1719-diag.zip
          17.66 MB
        7. 172.23.107.65-20210227-1719-diag.zip
          15.88 MB
        8. 172.23.121.213-20210227-1719-diag.zip
          18.96 MB
        9. 172.23.122.116-20210212-2218-diag.zip
          6.24 MB
        10. 172.23.122.117-20210212-2218-diag.zip
          12.01 MB
        11. 172.23.122.118-20210212-2218-diag.zip
          17.49 MB
        12. 172.23.122.156-20210217-0043-diag.zip
          25.08 MB
        13. 172.23.122.158-20210217-0043-diag.zip
          23.10 MB
        14. 172.23.122.159-20210217-0043-diag.zip
          20.12 MB
        15. 172.23.122.160-20210217-0043-diag.zip
          14.57 MB
        16. 172.23.122.161-20210217-0043-diag.zip
          23.98 MB
        17. 172.23.122.82-20210212-2218-diag.zip
          24.42 MB
        18. 172.23.122.90-20210212-2218-diag.zip
          34.40 MB
        19. 172.23.122.91-20210217-0043-diag.zip
          32.03 MB
        20. 172.23.122.92-20210217-0043-diag.zip
          18.98 MB
        21. 172.23.122.93-20210217-0043-diag.zip
          20.00 MB
        22. 172.23.122.97-20210212-2218-diag.zip
          41.19 MB
        23. 172.23.123.61-20210212-2218-diag.zip
          26.76 MB
        24. 172.23.123.62-20210212-2218-diag.zip
          28.60 MB
        25. image-2021-03-16-09-03-02-836.png
          image-2021-03-16-09-03-02-836.png
          133 kB
        26. screenshot-1.png
          screenshot-1.png
          18 kB
        27. screenshot-2.png
          screenshot-2.png
          42 kB
        28. screenshot-3.png
          screenshot-3.png
          19 kB
        29. test_2_4554.log
          259 kB
        30. test_4.log.txt
          408 kB
        31. test_5.log
          412 kB
        32. test_5.log.txt
          413 kB
        33. test.log
          412 kB
        No reviews matched the request. Check your Options in the drop-down menu of this sections header.

        Activity

          Build sync_gateway-3.0.0-223 contains go-couchbase commit 5740cd3 with commit message:
          MB-44328 tighten op retry, report error cause, go fmt

          build-team Couchbase Build Team added a comment - Build sync_gateway-3.0.0-223 contains go-couchbase commit 5740cd3 with commit message: MB-44328 tighten op retry, report error cause, go fmt

          Build sync_gateway-3.0.0-223 contains go-couchbase commit 5537227 with commit message:
          MB-44328, only give response Status in case of memcached errors

          build-team Couchbase Build Team added a comment - Build sync_gateway-3.0.0-223 contains go-couchbase commit 5537227 with commit message: MB-44328 , only give response Status in case of memcached errors

          Build sync_gateway-3.0.0-223 contains go-couchbase commit 1e26bfa with commit message:
          MB-44328 clarify memcached status in retry error

          build-team Couchbase Build Team added a comment - Build sync_gateway-3.0.0-223 contains go-couchbase commit 1e26bfa with commit message: MB-44328 clarify memcached status in retry error

          Build sync_gateway-3.0.0-223 contains go-couchbase commit 5bb57de with commit message:
          MB-44328 backoff on NMVB

          build-team Couchbase Build Team added a comment - Build sync_gateway-3.0.0-223 contains go-couchbase commit 5bb57de with commit message: MB-44328 backoff on NMVB

          Not seeing on 7.0.0-5017 run

          vikas.chaudhary Vikas Chaudhary added a comment - Not seeing on 7.0.0-5017 run

          People

            vikas.chaudhary Vikas Chaudhary
            vikas.chaudhary Vikas Chaudhary
            Votes:
            0 Vote for this issue
            Watchers:
            19 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved:

              PagerDuty