Details
-
Bug
-
Resolution: Fixed
-
Blocker
-
1.6.5.2
-
Security Level: Public
-
None
-
Ubuntu 1004 64bit
Description
Install ms 1.6.5.2r-15 on node 52.
Load 6M keys with 2K/key
Install ms 1.6.5.2r-15 on node 53
Add node 53 to node 52
Run a load on node 52 and 53.
Rebalance
After rebalance's done, stop the load on node 52 and 53
Kill vbucketmigrator on node 52 and 53
Replication failed to complete (after more than 10 hours)
curr_items: 6344000
curr_items_tot: 11935711
root@ubuntu1004-32:/opt/membase/bin/ep_engine/management# ./stats 10.2.1.52:11210 tap
ep_tap_ack_grace_period: 300
ep_tap_ack_interval: 1000
ep_tap_ack_window_size: 10
ep_tap_backoff_period: 1
ep_tap_bg_fetch_requeued: 0
ep_tap_bg_fetched: 7395838
ep_tap_bg_load_avg: 484
ep_tap_bg_max_load: 612491
ep_tap_bg_max_pending: 500
ep_tap_bg_max_wait: 100531147
ep_tap_bg_min_load: 5
ep_tap_bg_min_wait: 11
ep_tap_bg_num_samples: 1051838
ep_tap_bg_wait_avg: 211627
ep_tap_count: 1
ep_tap_deletes: 0
ep_tap_fg_fetched: 3429397
ep_tap_keepalive: 300
ep_tap_noop_interval: 20
ep_tap_throttled: 318404
ep_tap_total_fetched: 10828283
ep_tap_total_queue: 0
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:ack_log_size: 0
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:ack_playback_size: 0
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:ack_seqno: 4962841
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:ack_window_full: false
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:bg_backlog_size: 0
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:bg_jobs_completed: 4147034
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:bg_jobs_issued: 4147034
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:bg_queue_size: 0
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:bg_queued: 4147034
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:bg_result_size: 0
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:bg_results: 0
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:bg_wait_for_results: false
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:complete: false
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:connected: true
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:disconnects: 999
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:empty: true
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:flags: 20 (ack,vblist)
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:has_item: false
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:has_queued_item: false
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:idle: true
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:num_tap_nack: 722817
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:num_tap_tmpfail_survivors: 722817
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:paused: 1
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:pending_backfill: false
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:pending_disconnect: false
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:pending_disk_backfill: true
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:qlen: 0
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:qlen_high_pri: 0
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:qlen_low_pri: 0
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:rec_fetched: 1790841
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:reconnects: 999
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:recv_ack_seqno: 4962840
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:suspended: false
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:vb_filter:
eq_tapq:r-ns_1@10.2.1.53-ns_1@10.2.1.52-1299640803.679469:vb_filters: 512
root@ubuntu1004-32:/opt/membase/bin/ep_engine/management# ./stats 10.2.1.53:11210 tap
ep_tap_ack_grace_period: 300
ep_tap_ack_interval: 1000
ep_tap_ack_window_size: 10
ep_tap_backoff_period: 1
ep_tap_bg_fetch_requeued: 0
ep_tap_bg_fetched: 1832288
ep_tap_bg_load_avg: 38
ep_tap_bg_max_load: 203
ep_tap_bg_max_pending: 500
ep_tap_bg_max_wait: 12461217
ep_tap_bg_min_load: 15
ep_tap_bg_min_wait: 2720722
ep_tap_bg_num_samples: 452
ep_tap_bg_wait_avg: 7150421
ep_tap_count: 1
ep_tap_deletes: 0
ep_tap_fg_fetched: 2600900
ep_tap_keepalive: 300
ep_tap_noop_interval: 20
ep_tap_throttled: 1288511
ep_tap_total_fetched: 4434001
ep_tap_total_queue: 407177
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:ack_log_size: 0
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:ack_playback_size: 0
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:ack_seqno: 4434001
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:ack_window_full: true
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:bg_backlog_size: 1747341
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:bg_jobs_completed: 3172452
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:bg_jobs_issued: 3172452
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:bg_queue_size: 0
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:bg_queued: 3172452
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:bg_result_size: 1340164
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:bg_results: 0
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:bg_wait_for_results: false
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:complete: false
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:connected: true
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:disconnects: 999
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:empty: false
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:expires: 4294962427
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:flags: 20 (ack,vblist)
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:has_item: true
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:has_queued_item: true
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:idle: false
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:num_tap_nack: 318021
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:num_tap_tmpfail_survivors: 318021
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:paused: 1
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:pending_backfill: false
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:pending_disconnect: false
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:pending_disk_backfill: true
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:qlen: 407177
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:qlen_high_pri: 699
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:qlen_low_pri: 0
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:rec_fetched: 2602165
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:reconnects: 999
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:recv_ack_seqno: 4424000
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:suspended: false
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:vb_filter:
eq_tapq:r-ns_1@10.2.1.52-ns_1@10.2.1.52-1299640814.003399:vb_filters: 512