Details
-
Bug
-
Resolution: Duplicate
-
Blocker
-
3.0
-
None
-
Security Level: Public
-
None
Description
Large scale tests w/ toy-build. - 0.0.0-704
Load 240M items, resident at 4-7 percent active. 1 replica
Reboot entire cluster.
180M items after reboot.
All the warmup threads are complete.
[root@soursop-s11207 ~]# /opt/couchbase/bin/cbstats soursop-s11207.sc.couchbase.com:11210 raw warmup
ep_warmup: enabled
ep_warmup_dups: 0
ep_warmup_estimate_time: 8329436
ep_warmup_estimated_key_count: 392844712
ep_warmup_estimated_value_count: 0
ep_warmup_item_expired: 0
ep_warmup_key_count: 6573376
ep_warmup_keys_time: 20741092
ep_warmup_min_item_threshold: 100
ep_warmup_min_memory_threshold: 100
ep_warmup_oom: 0
ep_warmup_state: done
ep_warmup_thread: complete
ep_warmup_time: 175796881
ep_warmup_value_count: 6573376
[root@soursop-s11207 ~]# /opt/couchbase/bin/cbstats soursop-s11203.sc.couchbase.com:11210 raw warmup
ep_warmup: enabled
ep_warmup_dups: 332
ep_warmup_estimate_time: 7511414
ep_warmup_estimated_key_count: 494242728
ep_warmup_estimated_value_count: 0
ep_warmup_item_expired: 0
ep_warmup_key_count: 68648257
ep_warmup_keys_time: 20289368
ep_warmup_min_item_threshold: 100
ep_warmup_min_memory_threshold: 100
ep_warmup_oom: 0
ep_warmup_state: done
ep_warmup_thread: complete
ep_warmup_time: 3433636254
ep_warmup_value_count: 68648257
The curr_jtems are lesser
[root@soursop-s11207 ~]# /opt/couchbase/bin/cbstats soursop-s11207.sc.couchbase.com:11210 all | grep curr_items
curr_items: 13673991
curr_items_tot: 78364077
vb_active_curr_items: 13673991
vb_pending_curr_items: 0
vb_replica_curr_items: 64690086
[root@soursop-s11207 ~]# /opt/couchbase/bin/cbstats soursop-s11205.sc.couchbase.com:11210 all | grep curr_items
curr_items: 60113582
curr_items_tot: 96182272
vb_active_curr_items: 60113582
vb_pending_curr_items: 0
vb_replica_curr_items: 36068690
[root@soursop-s11207 ~]# /opt/couchbase/bin/cbstats soursop-s11203.sc.couchbase.com:11210 all | grep curr_items
curr_items: 52335342
curr_items_tot: 92185093
vb_active_curr_items: 52335342
vb_pending_curr_items: 0
vb_replica_curr_items: 39849751
[root@soursop-s11207 ~]#
Seeing Tap errors which may be unrelated -
[root@soursop-s11207 ~]# tail -f /opt/couchbase/var/lib/couchbase/logs/memcached.log.4.txt
Wed Oct 30 19:08:51.769748 PDT 3: (default) TAP (Producer) eq_tapq:replication_ns_1@soursop-s11205.sc.couchbase.com - Sending TAP_OPAQUE with command "close_backfill" and vbucket 228
Wed Oct 30 19:08:51.769887 PDT 3: (default) TAP (Producer) eq_tapq:replication_ns_1@soursop-s11205.sc.couchbase.com - Sending TAP_OPAQUE with command "close_backfill" and vbucket 231
Wed Oct 30 19:08:51.769990 PDT 3: (default) TAP (Producer) eq_tapq:replication_ns_1@soursop-s11205.sc.couchbase.com - Sending TAP_OPAQUE with command "close_backfill" and vbucket 234
Wed Oct 30 19:08:51.770084 PDT 3: (default) TAP (Producer) eq_tapq:replication_ns_1@soursop-s11205.sc.couchbase.com - Sending TAP_OPAQUE with command "close_backfill" and vbucket 240
Wed Oct 30 19:08:51.770185 PDT 3: (default) TAP (Producer) eq_tapq:replication_ns_1@soursop-s11205.sc.couchbase.com - Sending TAP_OPAQUE with command "close_backfill" and vbucket 243
Wed Oct 30 19:08:51.770332 PDT 3: (default) TAP (Producer) eq_tapq:replication_ns_1@soursop-s11205.sc.couchbase.com - Sending TAP_OPAQUE with command "close_backfill" and vbucket 246
Wed Oct 30 19:08:51.770441 PDT 3: (default) TAP (Producer) eq_tapq:replication_ns_1@soursop-s11205.sc.couchbase.com - Sending TAP_OPAQUE with command "close_backfill" and vbucket 249
Wed Oct 30 19:08:51.770540 PDT 3: (default) TAP (Producer) eq_tapq:replication_ns_1@soursop-s11205.sc.couchbase.com - Sending TAP_OPAQUE with command "close_backfill" and vbucket 252
Wed Oct 30 19:08:51.770631 PDT 3: (default) TAP (Producer) eq_tapq:replication_ns_1@soursop-s11205.sc.couchbase.com - Sending TAP_OPAQUE with command "close_backfill" and vbucket 255
Wed Oct 30 19:16:07.277800 PDT 3: 101 Closing connection due to read error: No route to host