Details
Description
I have a 4 node cluster, fully rebalanced, 1 bucket, 1 replica 1M items,
Checking stats, the cluster is fully balanced and replicated:
========curr itmes :===========
MACHINE: 10.2.1.50
curr_items: 249990
curr_items_tot: 499999
MACHINE: 10.2.1.12
curr_items: 249984
curr_items_tot: 500005
MACHINE: 10.2.1.13
curr_items: 250011
curr_items_tot: 499988
MACHINE: 10.2.1.14
curr_items: 250015
curr_items_tot: 500008
and aggregate moxi stats:
root@localhost ~]# /opt/membase/bin/mbstats 127.0.0.1:11211 all | grep curr
curr_items: 1000000
curr_items_tot: 2000000
but stats on http://10.2.1.12:8091/pools/default/buckets
shows: "replication":0.5 on two out for the four nodes, hence the warning that you might lose data if you failover.
here are the stats all from these two machines (10.2.1.12, 10.2.1.14)
Sharon-Barrs-MacBook-Pro:scripts sharonbarr$ ./stats 10.2.1.12:11210 all
accepting_conns: 1
auth_cmds: 1704
auth_errors: 0
bucket_active_conns: 1
bucket_conns: 53
bytes_read: 1816953547
bytes_written: 239436660
cas_badval: 0
cas_hits: 0
cas_misses: 0
cmd_flush: 0
cmd_get: 0
cmd_set: 0
conn_yields: 25688
connection_structures: 41
curr_connections: 20
curr_items: 249984
curr_items_tot: 500005
daemon_connections: 10
decr_hits: 0
decr_misses: 0
delete_hits: 0
delete_misses: 0
ep_bg_fetched: 0
ep_commit_num: 851
ep_commit_time: 0
ep_commit_time_total: 52
ep_data_age: 4
ep_data_age_highwat: 24
ep_db_cleaner_status: complete
ep_db_strategy: multiMTVBDB
ep_dbinit: 1
ep_dbname: /opt/membase/var/lib/membase/default-data/default
ep_dbshards: 4
ep_diskqueue_drain: 626334
ep_diskqueue_fill: 585432
ep_diskqueue_items: 739
ep_diskqueue_memory: 59120
ep_diskqueue_pending: 1640908
ep_expired: 0
ep_flush_duration: 0
ep_flush_duration_highwat: 14
ep_flush_duration_total: 80
ep_flush_preempts: 0
ep_flusher_state: running
ep_flusher_todo: 0
ep_io_num_read: 0
ep_io_num_write: 752600
ep_io_read_bytes: 0
ep_io_write_bytes: 1526563497
ep_item_begin_failed: 0
ep_item_commit_failed: 0
ep_item_flush_expired: 0
ep_item_flush_failed: 0
ep_items_rm_from_checkpoints: 651025
ep_kv_size: 1064181479
ep_latency_arith_cmd: 0
ep_latency_get_cmd: 0
ep_latency_store_cmd: 0
ep_max_data_size: 7261388800
ep_max_txn_size: 1000
ep_mem_high_wat: 5446041600
ep_mem_low_wat: 4356833280
ep_min_data_age: 0
ep_num_active_non_resident: 0
ep_num_checkpoint_remover_runs: 450
ep_num_eject_failures: 0
ep_num_eject_replicas: 0
ep_num_expiry_pager_runs: 0
ep_num_non_resident: 0
ep_num_not_my_vbuckets: 0
ep_num_pager_runs: 0
ep_num_value_ejects: 0
ep_onlineupdate: false
ep_onlineupdate_revert_add: 0
ep_onlineupdate_revert_delete: 0
ep_onlineupdate_revert_update: 0
ep_oom_errors: 0
ep_overhead: 13296296
ep_pending_ops: 0
ep_pending_ops_max: 0
ep_pending_ops_max_duration: 0
ep_pending_ops_total: 0
ep_queue_age_cap: 900
ep_queue_size: 0
ep_storage_age: 0
ep_storage_age_highwat: 24
ep_storage_type: featured
ep_store_max_concurrency: 10
ep_store_max_readers: 9
ep_store_max_readwrite: 1
ep_tap_bg_fetch_requeued: 0
ep_tap_bg_fetched: 0
ep_tap_keepalive: 300
ep_tmp_oom_errors: 0
ep_too_old: 0
ep_too_young: 0
ep_total_cache_size: 1063466723
ep_total_del_items: 0
ep_total_enqueued: 753380
ep_total_new_items: 667953
ep_total_persisted: 752600
ep_vb_total: 512
ep_vbucket_del: 172
ep_vbucket_del_avg_walltime: 6755
ep_vbucket_del_fail: 0
ep_vbucket_del_max_walltime: 9866
ep_vbucket_del_total_walltime: 1161932
ep_version: 1.6.5.3_211_g9531144
ep_warmed_up: 0
ep_warmup: true
ep_warmup_dups: 0
ep_warmup_oom: 0
ep_warmup_thread: complete
ep_warmup_time: 17776
get_hits: 0
get_misses: 0
incr_hits: 0
incr_misses: 0
libevent: 2.0.7-rc
limit_maxbytes: 67108864
listen_disabled_num: 0
mem_used: 1077477775
pid: 27595
pointer_size: 64
rejected_conns: 0
rusage_system: 72.270013
rusage_user: 125.563911
tap_checkpoint_end_received: 257
tap_checkpoint_end_sent: 409
tap_checkpoint_start_received: 1171
tap_checkpoint_start_sent: 1420
tap_connect_received: 646
tap_mutation_received: 876875
tap_mutation_sent: 749582
tap_opaque_received: 2044
tap_opaque_sent: 1719
tap_vbucket_set_received: 854
tap_vbucket_set_sent: 345
threads: 4
time: 1303955939
total_connections: 1720
uptime: 2258
vb_active_curr_items: 249984
vb_active_eject: 0
vb_active_ht_memory: 6408192
vb_active_itm_memory: 531693760
vb_active_num: 256
vb_active_num_non_resident: 0
vb_active_ops_create: 249984
vb_active_ops_delete: 0
vb_active_ops_reject: 37130
vb_active_ops_update: 83579
vb_active_perc_mem_resident: 100
vb_active_queue_age: 9032000
vb_active_queue_drain: 370949
vb_active_queue_fill: 333577
vb_active_queue_memory: 320
vb_active_queue_pending: 28694
vb_active_queue_size: 4
vb_dead_num: 0
vb_pending_curr_items: 0
vb_pending_eject: 0
vb_pending_ht_memory: 0
vb_pending_itm_memory: 0
vb_pending_num: 0
vb_pending_num_non_resident: 0
vb_pending_ops_create: 0
vb_pending_ops_delete: 0
vb_pending_ops_reject: 0
vb_pending_ops_update: 0
vb_pending_perc_mem_resident: 0
vb_pending_queue_age: 0
vb_pending_queue_drain: 0
vb_pending_queue_fill: 0
vb_pending_queue_memory: 0
vb_pending_queue_pending: 0
vb_pending_queue_size: 0
vb_replica_curr_items: 250021
vb_replica_eject: 0
vb_replica_ht_memory: 6408192
vb_replica_itm_memory: 531772963
vb_replica_num: 256
vb_replica_num_non_resident: 0
vb_replica_ops_create: 250021
vb_replica_ops_delete: 0
vb_replica_ops_reject: 4040
vb_replica_ops_update: 1068
vb_replica_perc_mem_resident: 100
vb_replica_queue_age: 1634573000
vb_replica_queue_drain: 255385
vb_replica_queue_fill: 251855
vb_replica_queue_memory: 58800
vb_replica_queue_pending: 1612214
vb_replica_queue_size: 735
version: 1.4.4_451_gfd84269
Sharon-Barrs-MacBook-Pro:scripts sharonbarr$ ./stats 10.2.1.14:11210 all
accepting_conns: 1
auth_cmds: 777
auth_errors: 0
bucket_active_conns: 1
bucket_conns: 13
bytes_read: 2873947322
bytes_written: 66031192
cas_badval: 0
cas_hits: 0
cas_misses: 0
cmd_flush: 0
cmd_get: 0
cmd_set: 0
conn_yields: 39047
connection_structures: 33
curr_connections: 19
curr_items: 250015
curr_items_tot: 500008
daemon_connections: 10
decr_hits: 0
decr_misses: 0
delete_hits: 0
delete_misses: 0
ep_bg_fetched: 0
ep_commit_num: 1038
ep_commit_time: 0
ep_commit_time_total: 70
ep_data_age: 9
ep_data_age_highwat: 28
ep_db_cleaner_status: complete
ep_db_strategy: multiMTVBDB
ep_dbinit: 1
ep_dbname: /opt/membase/var/lib/membase/default-data/default
ep_dbshards: 4
ep_diskqueue_drain: 1001941
ep_diskqueue_fill: 940349
ep_diskqueue_items: 2648
ep_diskqueue_memory: 211840
ep_diskqueue_pending: 5609910
ep_expired: 0
ep_flush_duration: 0
ep_flush_duration_highwat: 17
ep_flush_duration_total: 109
ep_flush_preempts: 0
ep_flusher_state: running
ep_flusher_todo: 0
ep_io_num_read: 1024
ep_io_num_write: 931809
ep_io_read_bytes: 0
ep_io_write_bytes: 1890072592
ep_item_begin_failed: 0
ep_item_commit_failed: 0
ep_item_flush_expired: 0
ep_item_flush_failed: 0
ep_items_rm_from_checkpoints: 500520
ep_kv_size: 1063475185
ep_latency_arith_cmd: 0
ep_latency_get_cmd: 0
ep_latency_store_cmd: 0
ep_max_data_size: 7261388800
ep_max_txn_size: 1000
ep_mem_high_wat: 5446041600
ep_mem_low_wat: 4356833280
ep_min_data_age: 0
ep_num_active_non_resident: 0
ep_num_checkpoint_remover_runs: 360
ep_num_eject_failures: 0
ep_num_eject_replicas: 0
ep_num_expiry_pager_runs: 0
ep_num_non_resident: 0
ep_num_not_my_vbuckets: 0
ep_num_pager_runs: 0
ep_num_value_ejects: 0
ep_onlineupdate: false
ep_onlineupdate_revert_add: 0
ep_onlineupdate_revert_delete: 0
ep_onlineupdate_revert_update: 0
ep_oom_errors: 0
ep_overhead: 13296216
ep_pending_ops: 0
ep_pending_ops_max: 0
ep_pending_ops_max_duration: 0
ep_pending_ops_total: 0
ep_queue_age_cap: 900
ep_queue_size: 0
ep_storage_age: 0
ep_storage_age_highwat: 79
ep_storage_type: featured
ep_store_max_concurrency: 10
ep_store_max_readers: 9
ep_store_max_readwrite: 1
ep_tap_bg_fetch_requeued: 0
ep_tap_bg_fetched: 0
ep_tap_keepalive: 300
ep_tmp_oom_errors: 0
ep_too_old: 0
ep_too_young: 0
ep_total_cache_size: 1063474672
ep_total_del_items: 0
ep_total_enqueued: 940349
ep_total_new_items: 500008
ep_total_persisted: 931809
ep_vb_total: 512
ep_vbucket_del: 1024
ep_vbucket_del_avg_walltime: 1055
ep_vbucket_del_fail: 0
ep_vbucket_del_max_walltime: 5499
ep_vbucket_del_total_walltime: 1081104
ep_version: 1.6.5.3_211_g9531144
ep_warmed_up: 0
ep_warmup: true
ep_warmup_dups: 0
ep_warmup_oom: 0
ep_warmup_thread: complete
ep_warmup_time: 59422
get_hits: 0
get_misses: 0
incr_hits: 0
incr_misses: 0
libevent: 2.0.7-rc
limit_maxbytes: 67108864
listen_disabled_num: 0
mem_used: 1076771401
pid: 4960
pointer_size: 64
rejected_conns: 0
rusage_system: 56.882352
rusage_user: 105.676934
tap_checkpoint_end_received: 104
tap_checkpoint_end_sent: 86
tap_checkpoint_start_received: 649
tap_checkpoint_start_sent: 382
tap_connect_received: 259
tap_mutation_received: 1388931
tap_mutation_sent: 154103
tap_opaque_received: 1284
tap_opaque_sent: 774
tap_vbucket_set_received: 512
threads: 4
time: 1303955944
total_connections: 791
uptime: 1808
vb_active_curr_items: 250015
vb_active_eject: 0
vb_active_ht_memory: 6408192
vb_active_itm_memory: 531759625
vb_active_num: 256
vb_active_num_non_resident: 0
vb_active_ops_create: 250015
vb_active_ops_delete: 0
vb_active_ops_reject: 24130
vb_active_ops_update: 770
vb_active_perc_mem_resident: 100
vb_active_queue_age: 0
vb_active_queue_drain: 275339
vb_active_queue_fill: 250785
vb_active_queue_memory: 0
vb_active_queue_pending: 0
vb_active_queue_size: 0
vb_dead_num: 0
vb_pending_curr_items: 0
vb_pending_eject: 0
vb_pending_ht_memory: 0
vb_pending_itm_memory: 0
vb_pending_num: 0
vb_pending_num_non_resident: 0
vb_pending_ops_create: 0
vb_pending_ops_delete: 0
vb_pending_ops_reject: 0
vb_pending_ops_update: 0
vb_pending_perc_mem_resident: 0
vb_pending_queue_age: 0
vb_pending_queue_drain: 0
vb_pending_queue_fill: 0
vb_pending_queue_memory: 0
vb_pending_queue_pending: 0
vb_pending_queue_size: 0
vb_replica_curr_items: 249993
vb_replica_eject: 0
vb_replica_ht_memory: 6408192
vb_replica_itm_memory: 531715047
vb_replica_num: 256
vb_replica_num_non_resident: 0
vb_replica_ops_create: 249993
vb_replica_ops_delete: 0
vb_replica_ops_reject: 44399
vb_replica_ops_update: 431031
vb_replica_perc_mem_resident: 100
vb_replica_queue_age: 4775846000
vb_replica_queue_drain: 726602
vb_replica_queue_fill: 689564
vb_replica_queue_memory: 211840
vb_replica_queue_pending: 5609910
vb_replica_queue_size: 2648
version: 1.4.4_451_gfd84269
Sharon-Barrs-MacBook-Pro:scripts sharonbarr$