Uploaded image for project: 'Couchbase Server'
  1. Couchbase Server
  2. MB-3497

replication does not finish after vbucketmigrator killed several times

    XMLWordPrintable

Details

    • Bug
    • Resolution: Fixed
    • Blocker
    • 1.6.5.2
    • 1.6.5.2
    • couchbase-bucket
    • Security Level: Public
    • None
    • Ubuntu 1004 64bit

    Description

      Install ms 1.6.5.2r-15 on node 52.
      Load 6M keys with 2K/key
      Install ms 1.6.5.2r-15 on node 53
      Add node 53 to node 52
      Run a load on node 52 and 53.
      Rebalance
      After rebalance's done, stop the load on node 52 and 53
      Kill vbucketmigrator on node 52 and 53
      Replication failed to complete (after more than 10 hours)
      curr_items: 6344000
      curr_items_tot: 11935711

      root@ubuntu1004-32:/opt/membase/bin/ep_engine/management# ./stats 10.2.1.52:11210 tap
      ep_tap_ack_grace_period: 300
      ep_tap_ack_interval: 1000
      ep_tap_ack_window_size: 10
      ep_tap_backoff_period: 1
      ep_tap_bg_fetch_requeued: 0
      ep_tap_bg_fetched: 7395838
      ep_tap_bg_load_avg: 484
      ep_tap_bg_max_load: 612491
      ep_tap_bg_max_pending: 500
      ep_tap_bg_max_wait: 100531147
      ep_tap_bg_min_load: 5
      ep_tap_bg_min_wait: 11
      ep_tap_bg_num_samples: 1051838
      ep_tap_bg_wait_avg: 211627
      ep_tap_count: 1
      ep_tap_deletes: 0
      ep_tap_fg_fetched: 3429397
      ep_tap_keepalive: 300
      ep_tap_noop_interval: 20
      ep_tap_throttled: 318404
      ep_tap_total_fetched: 10828283
      ep_tap_total_queue: 0
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:ack_log_size: 0
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:ack_playback_size: 0
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:ack_seqno: 4962841
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:ack_window_full: false
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:bg_backlog_size: 0
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:bg_jobs_completed: 4147034
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:bg_jobs_issued: 4147034
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:bg_queue_size: 0
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:bg_queued: 4147034
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:bg_result_size: 0
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:bg_results: 0
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:bg_wait_for_results: false
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:complete: false
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:connected: true
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:disconnects: 999
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:empty: true
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:flags: 20 (ack,vblist)
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:has_item: false
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:has_queued_item: false
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:idle: true
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:num_tap_nack: 722817
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:num_tap_tmpfail_survivors: 722817
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:paused: 1
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:pending_backfill: false
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:pending_disconnect: false
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:pending_disk_backfill: true
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:qlen: 0
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:qlen_high_pri: 0
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:qlen_low_pri: 0
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:rec_fetched: 1790841
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:reconnects: 999
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:recv_ack_seqno: 4962840
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:suspended: false
      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:vb_filter:

      { [512,1023] }

      eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:vb_filters: 512

      root@ubuntu1004-32:/opt/membase/bin/ep_engine/management# ./stats 10.2.1.53:11210 tap
      ep_tap_ack_grace_period: 300
      ep_tap_ack_interval: 1000
      ep_tap_ack_window_size: 10
      ep_tap_backoff_period: 1
      ep_tap_bg_fetch_requeued: 0
      ep_tap_bg_fetched: 1832288
      ep_tap_bg_load_avg: 38
      ep_tap_bg_max_load: 203
      ep_tap_bg_max_pending: 500
      ep_tap_bg_max_wait: 12461217
      ep_tap_bg_min_load: 15
      ep_tap_bg_min_wait: 2720722
      ep_tap_bg_num_samples: 452
      ep_tap_bg_wait_avg: 7150421
      ep_tap_count: 1
      ep_tap_deletes: 0
      ep_tap_fg_fetched: 2600900
      ep_tap_keepalive: 300
      ep_tap_noop_interval: 20
      ep_tap_throttled: 1288511
      ep_tap_total_fetched: 4434001
      ep_tap_total_queue: 407177
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:ack_log_size: 0
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:ack_playback_size: 0
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:ack_seqno: 4434001
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:ack_window_full: true
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:bg_backlog_size: 1747341
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:bg_jobs_completed: 3172452
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:bg_jobs_issued: 3172452
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:bg_queue_size: 0
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:bg_queued: 3172452
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:bg_result_size: 1340164
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:bg_results: 0
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:bg_wait_for_results: false
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:complete: false
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:connected: true
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:disconnects: 999
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:empty: false
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:expires: 4294962427
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:flags: 20 (ack,vblist)
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:has_item: true
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:has_queued_item: true
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:idle: false
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:num_tap_nack: 318021
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:num_tap_tmpfail_survivors: 318021
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:paused: 1
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:pending_backfill: false
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:pending_disconnect: false
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:pending_disk_backfill: true
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:qlen: 407177
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:qlen_high_pri: 699
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:qlen_low_pri: 0
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:rec_fetched: 2602165
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:reconnects: 999
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:recv_ack_seqno: 4424000
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:suspended: false
      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:vb_filter:

      { [0,511] }

      eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:vb_filters: 512

      Attachments

        No reviews matched the request. Check your Options in the drop-down menu of this sections header.

        Activity

          People

            trond Trond Norbye
            thuan Thuan Nguyen
            Votes:
            0 Vote for this issue
            Watchers:
            0 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved:

              Gerrit Reviews

                There are no open Gerrit changes

                PagerDuty