Description
I loaded a 2 node cluster with 27M keys, ~2K value size.
I killed vbm many times on 10.2.1.13 and then left it alone.
a day later, my replication is not done:
the current_itmes_tot on 10.2.1.14 is almost the same as the curr_itmes indicating that replication is stopped.
MACHINE: 10.2.1.13
curr_items: 13879727
curr_items_tot: 27759333
ep_mem_high_wat: 24499716096
ep_mem_low_wat: 19599772876
ep_num_eject_failures: 5066627
ep_num_eject_replicas: 11173526
ep_num_value_ejects: 17713396
mem_used: 20746175923
MACHINE: 10.2.1.14
curr_items: 13879606
curr_items_tot: 14930053
ep_mem_high_wat: 24499716096
ep_mem_low_wat: 19599772876
ep_num_eject_failures: 13562
ep_num_eject_replicas: 210318
ep_num_value_ejects: 2630302
mem_used: 23291577067
Below are the tap and all stats for both nods:
Sharon-Barrs-MacBook-Pro:scripts sharonbarr$ ./stats 10.2.1.13:11210 tap
ep_tap_ack_grace_period: 300
ep_tap_ack_interval: 1000
ep_tap_ack_window_size: 10
ep_tap_backoff_period: 1
ep_tap_bg_fetch_requeued: 0
ep_tap_bg_fetched: 787031
ep_tap_bg_max_pending: 500
ep_tap_count: 1
ep_tap_deletes: 0
ep_tap_fg_fetched: 1814926
ep_tap_keepalive: 300
ep_tap_noop_interval: 20
ep_tap_throttled: 21805895
ep_tap_total_fetched: 2604006
ep_tap_total_queue: 12894824
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:ack_log_size: 0
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:ack_playback_size: 0
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:ack_seqno: 1704131
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:ack_window_full: true
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:bg_backlog_size: 13439652
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:bg_jobs_completed: 926722
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:bg_jobs_issued: 926722
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:bg_queue_size: 0
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:bg_queued: 926722
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:bg_result_size: 544828
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:bg_results: 0
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:bg_wait_for_results: false
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:complete: false
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:connected: true
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:disconnects: 2573
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:empty: false
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:expires: 4294877627
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:flags: 20 (ack,vblist)
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:has_item: true
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:has_queued_item: true
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:idle: false
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:num_tap_nack: 146783
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:num_tap_tmpfail_survivors: 146783
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:paused: true
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:pending_backfill: false
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:pending_disconnect: false
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:pending_disk_backfill: true
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:qlen: 12894824
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:qlen_high_pri: 2573
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:qlen_low_pri: 0
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:rec_fetched: 1322237
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:reconnects: 2573
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:recv_ack_seqno: 1694130
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:suspended: false
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:vb_filter:
eq_tapq:r-ns_1@10.2.1.13-1299001872.432894:vb_filters: 512
Sharon-Barrs-MacBook-Pro:scripts sharonbarr$
Sharon-Barrs-MacBook-Pro:scripts sharonbarr$
Sharon-Barrs-MacBook-Pro:scripts sharonbarr$
Sharon-Barrs-MacBook-Pro:scripts sharonbarr$
Sharon-Barrs-MacBook-Pro:scripts sharonbarr$
Sharon-Barrs-MacBook-Pro:scripts sharonbarr$ ./stats 10.2.1.14:11210 tap
ep_tap_ack_grace_period: 300
ep_tap_ack_interval: 1000
ep_tap_ack_window_size: 10
ep_tap_backoff_period: 1
ep_tap_bg_fetch_requeued: 0
ep_tap_bg_fetched: 3680303
ep_tap_bg_load_avg: 796
ep_tap_bg_max_load: 1566336
ep_tap_bg_max_pending: 500
ep_tap_bg_max_wait: 37675026
ep_tap_bg_min_load: 19
ep_tap_bg_min_wait: 9
ep_tap_bg_num_samples: 2780696
ep_tap_bg_wait_avg: 82697
ep_tap_count: 1
ep_tap_deletes: 0
ep_tap_fg_fetched: 33127659
ep_tap_keepalive: 300
ep_tap_noop_interval: 20
ep_tap_throttled: 146923
ep_tap_total_fetched: 36808475
ep_tap_total_queue: 0
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:ack_log_size: 0
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:ack_playback_size: 0
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:ack_seqno: 36808475
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:ack_window_full: false
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:bg_backlog_size: 0
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:bg_jobs_completed: 3680303
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:bg_jobs_issued: 3680303
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:bg_queue_size: 0
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:bg_queued: 3680303
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:bg_result_size: 0
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:bg_results: 0
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:bg_wait_for_results: false
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:complete: false
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:connected: true
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:empty: true
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:flags: 20 (ack,vblist)
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:has_item: false
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:has_queued_item: false
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:idle: true
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:num_tap_nack: 21805895
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:num_tap_tmpfail_survivors: 21805895
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:paused: true
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:pending_backfill: false
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:pending_disconnect: false
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:pending_disk_backfill: true
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:qlen: 0
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:qlen_high_pri: 0
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:qlen_low_pri: 0
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:rec_fetched: 35908868
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:recv_ack_seqno: 36808474
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:suspended: false
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:vb_filter:
eq_tapq:r-ns_1@10.2.1.13-1299001873.481843:vb_filters: 512
Sharon-Barrs-MacBook-Pro:scripts sharonbarr$
Sharon-Barrs-MacBook-Pro:scripts sharonbarr$ ./stats 10.2.1.13:11210 all
auth_cmds: 3185
auth_errors: 0
bucket_conns: 7
bytes_read: 93658809634
bytes_written: 2679020999
cas_badval: 0
cas_hits: 0
cas_misses: 0
cmd_flush: 0
cmd_get: 81786
cmd_set: 15143130
conn_yields: 21558
connection_structures: 21
curr_connections: 17
curr_items: 13879727
curr_items_tot: 27759333
daemon_connections: 10
decr_hits: 0
decr_misses: 0
delete_hits: 0
delete_misses: 0
ep_bg_fetched: 0
ep_commit_num: 30805
ep_commit_time: 0
ep_commit_time_total: 7464
ep_data_age: 269
ep_data_age_highwat: 894
ep_db_cleaner_status: complete
ep_db_strategy: multiMTVBDB
ep_dbinit: 1
ep_dbname: /var/opt/membase/1.6.5.2r/data/ns_1/default
ep_dbshards: 4
ep_expired: 0
ep_flush_duration: 97
ep_flush_duration_highwat: 409
ep_flush_duration_total: 12769
ep_flush_preempts: 0
ep_flusher_state: running
ep_flusher_todo: 0
ep_io_num_read: 1331859
ep_io_num_write: 29138846
ep_io_read_bytes: 2349700369
ep_io_write_bytes: 51420254709
ep_item_begin_failed: 0
ep_item_commit_failed: 0
ep_item_flush_expired: 0
ep_item_flush_failed: 0
ep_kv_size: 20544275395
ep_max_data_size: 32666288128
ep_max_txn_size: 1000
ep_mem_high_wat: 24499716096
ep_mem_low_wat: 19599772876
ep_min_data_age: 0
ep_num_active_non_resident: 6539870
ep_num_eject_failures: 5066627
ep_num_eject_replicas: 11173526
ep_num_expiry_pager_runs: 5
ep_num_non_resident: 17705330
ep_num_not_my_vbuckets: 1831
ep_num_pager_runs: 2
ep_num_value_ejects: 17713396
ep_oom_errors: 0
ep_overhead: 201900528
ep_pending_ops: 0
ep_pending_ops_max: 0
ep_pending_ops_max_duration: 0
ep_pending_ops_total: 0
ep_queue_age_cap: 900
ep_queue_size: 0
ep_storage_age: 0
ep_storage_age_highwat: 892
ep_storage_type: featured
ep_store_max_concurrency: 10
ep_store_max_readers: 9
ep_store_max_readwrite: 1
ep_tap_bg_fetch_requeued: 0
ep_tap_bg_fetched: 787031
ep_tap_keepalive: 300
ep_tmp_oom_errors: 278578
ep_too_old: 0
ep_too_young: 0
ep_total_cache_size: 51467805435
ep_total_del_items: 0
ep_total_enqueued: 29158519
ep_total_new_items: 28232089
ep_total_persisted: 29136359
ep_vbucket_del: 203
ep_vbucket_del_avg_walltime: 54311421111
ep_vbucket_del_fail: 0
ep_vbucket_del_max_walltime: 89379539186
ep_vbucket_del_total_walltime: 11025218485601
ep_version: 1.6.5.2r_7_g2185f8e
ep_warmed_up: 0
ep_warmup: true
ep_warmup_dups: 0
ep_warmup_oom: 0
ep_warmup_thread: complete
ep_warmup_time: 11832
get_hits: 81786
get_misses: 0
incr_hits: 0
incr_misses: 0
libevent: 1.4.13-stable
limit_maxbytes: 67108864
mem_used: 20746175923
pid: 1031
pointer_size: 64
rejected_conns: 0
rusage_system: 4369.814687
rusage_user: 1395.486853
tap_connect_received: 3086
tap_mutation_received: 36807962
tap_mutation_sent: 2601957
tap_opaque_received: 513
tap_opaque_sent: 1025
tap_vbucket_set_sent: 1024
threads: 4
time: 1299091353
total_connections: 3269
uptime: 89703
version: 1.4.4_374_g270f398
Sharon-Barrs-MacBook-Pro:scripts sharonbarr$ ./stats 10.2.1.14:11210 al
Sharon-Barrs-MacBook-Pro:scripts sharonbarr$
Sharon-Barrs-MacBook-Pro:scripts sharonbarr$
Sharon-Barrs-MacBook-Pro:scripts sharonbarr$
Sharon-Barrs-MacBook-Pro:scripts sharonbarr$
Sharon-Barrs-MacBook-Pro:scripts sharonbarr$
Sharon-Barrs-MacBook-Pro:scripts sharonbarr$ ./stats 10.2.1.14:11210 all
auth_cmds: 3115
auth_errors: 0
bucket_conns: 10
bytes_read: 29614209176
bytes_written: 1777696625
cas_badval: 0
cas_hits: 0
cas_misses: 0
cmd_flush: 0
cmd_get: 79061
cmd_set: 13874743
conn_yields: 856
connection_structures: 20
curr_connections: 20
curr_items: 13879606
curr_items_tot: 14930053
daemon_connections: 10
decr_hits: 0
decr_misses: 0
delete_hits: 0
delete_misses: 0
ep_bg_fetched: 99
ep_bg_load_avg: 11203
ep_bg_max_load: 400366
ep_bg_max_wait: 383150
ep_bg_min_load: 41
ep_bg_min_wait: 22
ep_bg_num_samples: 99
ep_bg_wait_avg: 25161
ep_commit_num: 50997
ep_commit_time: 0
ep_commit_time_total: 7783
ep_data_age: 0
ep_data_age_highwat: 196
ep_db_cleaner_status: complete
ep_db_strategy: multiMTVBDB
ep_dbinit: 0
ep_dbname: /var/opt/membase/1.6.5.2r/data/ns_1/default
ep_dbshards: 4
ep_expired: 0
ep_flush_duration: 0
ep_flush_duration_highwat: 100
ep_flush_duration_total: 8102
ep_flush_preempts: 0
ep_flusher_state: running
ep_flusher_todo: 0
ep_io_num_read: 3680402
ep_io_num_write: 15657849
ep_io_read_bytes: 6495758159
ep_io_write_bytes: 27632235537
ep_item_begin_failed: 0
ep_item_commit_failed: 0
ep_item_flush_expired: 0
ep_item_flush_failed: 0
ep_kv_size: 23177707771
ep_max_data_size: 32666288128
ep_max_txn_size: 1000
ep_mem_high_wat: 24499716096
ep_mem_low_wat: 19599772876
ep_min_data_age: 0
ep_num_active_non_resident: 2361879
ep_num_eject_failures: 13562
ep_num_eject_replicas: 210318
ep_num_expiry_pager_runs: 24
ep_num_non_resident: 2572197
ep_num_not_my_vbuckets: 0
ep_num_pager_runs: 1
ep_num_value_ejects: 2630302
ep_oom_errors: 0
ep_overhead: 113869296
ep_pending_ops: 0
ep_pending_ops_max: 2
ep_pending_ops_max_duration: 117136
ep_pending_ops_total: 12
ep_queue_age_cap: 900
ep_queue_size: 0
ep_storage_age: 0
ep_storage_age_highwat: 194
ep_storage_type: featured
ep_store_max_concurrency: 10
ep_store_max_readers: 9
ep_store_max_readwrite: 1
ep_tap_bg_fetch_requeued: 0
ep_tap_bg_fetched: 3680303
ep_tap_keepalive: 300
ep_tmp_oom_errors: 0
ep_too_old: 0
ep_too_young: 0
ep_total_cache_size: 27769371267
ep_total_del_items: 0
ep_total_enqueued: 15713335
ep_total_new_items: 14930053
ep_total_persisted: 15657849
ep_vbucket_del: 0
ep_vbucket_del_fail: 0
ep_version: 1.6.5.2r_7_g2185f8e
ep_warmed_up: 0
ep_warmup: true
ep_warmup_dups: 0
ep_warmup_oom: 0
ep_warmup_thread: complete
ep_warmup_time: 12520
get_hits: 79060
get_misses: 1
incr_hits: 0
incr_misses: 0
libevent: 1.4.13-stable
limit_maxbytes: 67108864
mem_used: 23291577067
pid: 28989
pointer_size: 64
rejected_conns: 0
rusage_system: 1289.092028
rusage_user: 1246.220545
tap_connect_received: 1
tap_mutation_received: 2592097
tap_mutation_sent: 36807962
tap_opaque_received: 1025
tap_opaque_sent: 513
tap_vbucket_set_received: 1024
threads: 4
time: 1299091360
total_connections: 3138
uptime: 89556
version: 1.4.4_374_g270f398
Sharon-Barrs-MacBook-Pro:scripts sharonbarr$