Flusher queue doesn't get flushed

Scenario
--------------

• flusher queue always has a few items and never goes to zero even after minutes of waiting
• seen both on cluster_run and servers on vms
• reproducible consistently with
  ./testrunner -i cluster_run.ini -t xdcr.uniXDCR.unidirectional.load_with_async_ops,items=1000,rdirection=unidirection,ctopology=chain,doc-ops=delete-delete
• seen with 1000, 100 items
• not seen in some cases like
  ./testrunner -i cluster_run.ini -t xdcr.uniXDCR.unidirectional.load_with_ops,replicas=1,items=10000,value_size=128,ctopology=chain,rdirection=unidirection,doc-ops=update-delete
• consistently seen after warmup
• all tests that wait for final verification after drain queue size becomes 0 are timing out.

Setup
---------
Source cluster : 10.3.4.186, 10.3.4.187
Destination : 10.3.4.188, 10.3.4.189

setup live at 10.3.4.187:8091 , default login credentials for SSH and GUI.

GDB info on .187
----------------------------
Thread 13 (Thread 0x7effd79ad700 (LWP 4381)):
#0 0x00007effd904174d in read () from /lib64/libc.so.6
#1 0x00007effd8fd7fe8 in _IO_new_file_underflow () from /lib64/libc.so.6
#2 0x00007effd8fd9aee in _IO_default_uflow_internal () from /lib64/libc.so.6
#3 0x00007effd8fce1ca in _IO_getline_info_internal () from /lib64/libc.so.6
#4 0x00007effd8fcd029 in fgets () from /lib64/libc.so.6
#5 0x00007effd79ae8b1 in check_stdin_thread (arg=<value optimized out>) at /home/buildbot/centos-5-x64-300-builder/build/build/memcached/extensions/daemon/stdin_check.c:38
#6 0x00007effdb0f7b6f in platform_thread_wrap (arg=0x29d4070) at /home/buildbot/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19
#7 0x00007effd9ead9d1 in start_thread () from /lib64/libpthread.so.0
#8 0x00007effd904eb6d in clone () from /lib64/libc.so.6

Thread 12 (Thread 0x7effd6d98700 (LWP 4382)):
#0 0x00007effd9eb198e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007effdb0f78eb in cb_cond_timedwait (cond=0x7effd6face60, mutex=0x7effd6face20, ms=<value optimized out>) at /home/buildbot/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:156
#2 0x00007effd6d9c548 in logger_thead_main (arg=0x2a14b00) at /home/buildbot/centos-5-x64-300-builder/build/build/memcached/extensions/loggers/file_logger.c:372
#3 0x00007effdb0f7b6f in platform_thread_wrap (arg=0x29d4080) at /home/buildbot/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19
#4 0x00007effd9ead9d1 in start_thread () from /lib64/libpthread.so.0
#5 0x00007effd904eb6d in clone () from /lib64/libc.so.6

Thread 11 (Thread 0x7effd618a700 (LWP 4383)):
#0 0x00007effd904f163 in epoll_wait () from /lib64/libc.so.6
#1 0x00007effda68e376 in epoll_dispatch (base=0xbd60500, tv=<value optimized out>) at epoll.c:404
#2 0x00007effda679c44 in event_base_loop (base=0xbd60500, flags=<value optimized out>) at event.c:1558
#3 0x00007effdb0f7b6f in platform_thread_wrap (arg=0x29d4190) at /home/buildbot/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19
#4 0x00007effd9ead9d1 in start_thread () from /lib64/libpthread.so.0
#5 0x00007effd904eb6d in clone () from /lib64/libc.so.6

Thread 10 (Thread 0x7effd5789700 (LWP 4384)):
#0 0x00007effd904f163 in epoll_wait () from /lib64/libc.so.6
#1 0x00007effda68e376 in epoll_dispatch (base=0xbd60280, tv=<value optimized out>) at epoll.c:404
#2 0x00007effda679c44 in event_base_loop (base=0xbd60280, flags=<value optimized out>) at event.c:1558
#3 0x00007effdb0f7b6f in platform_thread_wrap (arg=0x29d4180) at /home/buildbot/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19
#4 0x00007effd9ead9d1 in start_thread () from /lib64/libpthread.so.0
#5 0x00007effd904eb6d in clone () from /lib64/libc.so.6

Thread 9 (Thread 0x7effd4d88700 (LWP 4385)):
#0 0x00007effd904f163 in epoll_wait () from /lib64/libc.so.6
#1 0x00007effda68e376 in epoll_dispatch (base=0xbd60c80, tv=<value optimized out>) at epoll.c:404
#2 0x00007effda679c44 in event_base_loop (base=0xbd60c80, flags=<value optimized out>) at event.c:1558
#3 0x00007effdb0f7b6f in platform_thread_wrap (arg=0x29d4170) at /home/buildbot/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19
#4 0x00007effd9ead9d1 in start_thread () from /lib64/libpthread.so.0
#5 0x00007effd904eb6d in clone () from /lib64/libc.so.6

Thread 8 (Thread 0x7effd4387700 (LWP 4386)):
#0 0x00007effd904f163 in epoll_wait () from /lib64/libc.so.6
#1 0x00007effda68e376 in epoll_dispatch (base=0xbd60a00, tv=<value optimized out>) at epoll.c:404
#2 0x00007effda679c44 in event_base_loop (base=0xbd60a00, flags=<value optimized out>) at event.c:1558
#3 0x00007effdb0f7b6f in platform_thread_wrap (arg=0x29d4160) at /home/buildbot/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19
#4 0x00007effd9ead9d1 in start_thread () from /lib64/libpthread.so.0
#5 0x00007effd904eb6d in clone () from /lib64/libc.so.6

[root@centos-64-x64 ~]# /opt/couchbase/bin/cbstats localhost:11210 all
accepting_conns: 1
auth_cmds: 6
auth_errors: 0
bucket_active_conns: 1
bucket_conns: 13
bytes: 35154528
bytes_read: 1277423
bytes_written: 300104578
cas_badval: 0
cas_hits: 0
cas_misses: 0
cmd_flush: 0
cmd_get: 0
cmd_set: 500
conn_yields: 85
connection_structures: 10500
curr_connections: 17
curr_conns_on_port_11207: 2
curr_conns_on_port_11209: 12
curr_conns_on_port_11210: 3
curr_items: 350
curr_items_tot: 700
curr_temp_items: 0
daemon_connections: 6
decr_hits: 0
decr_misses: 0
delete_hits: 150
delete_misses: 0
ep_access_scanner_last_runtime: 0
ep_access_scanner_num_items: 0
ep_access_scanner_task_time: 2014-04-17 23:30:57
ep_allow_data_loss_during_shutdown: 1
ep_alog_block_size: 4096
ep_alog_path: /opt/couchbase/var/lib/couchbase/data/default/access.log
ep_alog_sleep_time: 1440
ep_alog_task_time: 10
ep_backend: couchdb
ep_bg_fetch_delay: 0
ep_bg_fetched: 0
ep_bg_meta_fetched: 0
ep_bg_remaining_jobs: 0
ep_bucket_priority: LOW
ep_chk_max_items: 5000
ep_chk_period: 1800
ep_chk_persistence_remains: 0
ep_chk_persistence_timeout: 10
ep_chk_remover_stime: 5
ep_commit_num: 2378
ep_commit_time: 0
ep_commit_time_total: 5599
ep_config_file:
ep_conflict_resolution_type: seqno
ep_couch_bucket: default
ep_couch_host: 127.0.0.1
ep_couch_port: 11213
ep_couch_reconnect_sleeptime: 250
ep_couch_response_timeout: 180000
ep_data_traffic_enabled: 0
ep_db_data_size: 274320
ep_db_file_size: 16666010
ep_dbname: /opt/couchbase/var/lib/couchbase/data/default
ep_degraded_mode: 0
ep_diskqueue_drain: 1272
ep_diskqueue_fill: 1290
ep_diskqueue_items: 18
ep_diskqueue_memory: 1296
ep_diskqueue_pending: 9036
ep_exp_pager_stime: 3600
ep_expired_access: 0
ep_expired_pager: 0
ep_failpartialwarmup: 0
ep_flush_all: false
ep_flush_duration_total: 8
ep_flushall_enabled: 0
ep_flusher_state: running
ep_flusher_todo: 0
ep_getl_default_timeout: 15
ep_getl_max_timeout: 30
ep_ht_locks: 5
ep_ht_size: 3079
ep_initfile:
ep_io_num_read: 0
ep_io_num_write: 1265
ep_io_read_bytes: 0
ep_io_write_bytes: 261510
ep_item_begin_failed: 0
ep_item_commit_failed: 0
ep_item_eviction_policy: value_only
ep_item_flush_expired: 0
ep_item_flush_failed: 0
ep_item_num_based_new_chk: 1
ep_items_rm_from_checkpoints: 2402
ep_keep_closed_chks: 0
ep_kv_size: 235440
ep_max_bg_remaining_jobs: 0
ep_max_checkpoints: 2
ep_max_failover_entries: 25
ep_max_item_size: 20971520
ep_max_num_shards: 4
ep_max_num_workers: 3
ep_max_size: 2169503744
ep_max_threads: 0
ep_max_vbuckets: 1024
ep_mem_high_wat: 1844078182
ep_mem_low_wat: 1627127808
ep_mem_tracker_enabled: true
ep_meta_data_memory: 46090
ep_mlog_compactor_runs: 0
ep_mutation_mem_threshold: 95
ep_num_access_scanner_runs: 0
ep_num_eject_failures: 0
ep_num_expiry_pager_runs: 2
ep_num_non_resident: 0
ep_num_not_my_vbuckets: 0
ep_num_ops_del_meta: 0
ep_num_ops_del_meta_res_fail: 0
ep_num_ops_del_ret_meta: 0
ep_num_ops_get_meta: 0
ep_num_ops_get_meta_on_set_meta: 0
ep_num_ops_set_meta: 0
ep_num_ops_set_meta_res_fail: 0
ep_num_ops_set_ret_meta: 0
ep_num_pager_runs: 0
ep_num_value_ejects: 0
ep_num_workers: 4
ep_oom_errors: 0
ep_overhead: 27466374
ep_pager_active_vb_pcnt: 40
ep_pending_compactions: 0
ep_pending_ops: 0
ep_pending_ops_max: 0
ep_pending_ops_max_duration: 0
ep_pending_ops_total: 0
ep_postInitfile:
ep_queue_size: 18
ep_rollback_count: 0
ep_startup_time: 1397691056
ep_storage_age: 0
ep_storage_age_highwat: 1
ep_tap_ack_grace_period: 300
ep_tap_ack_initial_sequence_number: 1
ep_tap_ack_interval: 1000
ep_tap_ack_window_size: 10
ep_tap_backfill_resident: 0.9
ep_tap_backlog_limit: 5000
ep_tap_backoff_period: 5
ep_tap_bg_fetch_requeued: 0
ep_tap_bg_fetched: 0
ep_tap_bg_max_pending: 500
ep_tap_keepalive: 300
ep_tap_noop_interval: 20
ep_tap_requeue_sleep_time: 0.1
ep_tap_throttle_cap_pcnt: 10
ep_tap_throttle_queue_cap: 1000000
ep_tap_throttle_threshold: 90
ep_tmp_oom_errors: 0
ep_total_cache_size: 226690
ep_total_del_items: 265
ep_total_enqueued: 1290
ep_total_new_items: 965
ep_total_persisted: 1230
ep_uncommitted_items: 0
ep_uuid: 28b4c2f6d709668a060c9c6489f4003d
ep_value_size: 189350
ep_vb0: 0
ep_vb_snapshot_total: 1704
ep_vb_total: 1024
ep_vbucket_del: 512
ep_vbucket_del_avg_walltime: 67098
ep_vbucket_del_fail: 0
ep_vbucket_del_max_walltime: 2666206
ep_version: 2.1.1r-602-g0e4754a
ep_waitforwarmup: 0
ep_warmup: 1
ep_warmup_batch_size: 1000
ep_warmup_dups: 0
ep_warmup_min_items_threshold: 100
ep_warmup_min_memory_threshold: 100
ep_warmup_oom: 0
ep_warmup_thread: complete
ep_warmup_time: 461934
get_hits: 0
get_misses: 0
incr_hits: 0
incr_misses: 0
libevent: 2.0.11-stable
listen_disabled_num: 0
max_conns_on_port_11207: 10000
max_conns_on_port_11209: 1000
max_conns_on_port_11210: 10000
mem_used: 35154528
memcached_version: 2.0.1-macosx-171-g493f088
pid: 4380
pointer_size: 64
rejected_conns: 0
rusage_system: 265.075702
rusage_user: 860.965113
tap_checkpoint_end_received: 335
tap_checkpoint_end_sent: 322
tap_checkpoint_start_received: 847
tap_checkpoint_start_sent: 1346
tap_connect_received: 2
tap_delete_received: 150
tap_delete_sent: 150
tap_mutation_received: 500
tap_mutation_sent: 500
tap_opaque_received: 1026
tap_opaque_sent: 1028
threads: 4
time: 1397698341
total_connections: 99
uptime: 7316
vb_active_curr_items: 350
vb_active_eject: 0
vb_active_expired: 0
vb_active_ht_memory: 12849152
vb_active_itm_memory: 113345
vb_active_meta_data_memory: 23045
vb_active_num: 512
vb_active_num_non_resident: 0
vb_active_ops_create: 468
vb_active_ops_delete: 118
vb_active_ops_reject: 0
vb_active_ops_update: 0
vb_active_perc_mem_resident: 100
vb_active_queue_age: 115921000
vb_active_queue_drain: 625
vb_active_queue_fill: 641
vb_active_queue_memory: 1152
vb_active_queue_pending: 8062
vb_active_queue_size: 16
vb_dead_num: 0
vb_pending_curr_items: 0
vb_pending_eject: 0
vb_pending_expired: 0
vb_pending_ht_memory: 0
vb_pending_itm_memory: 0
vb_pending_meta_data_memory: 0
vb_pending_num: 0
vb_pending_num_non_resident: 0
vb_pending_ops_create: 0
vb_pending_ops_delete: 0
vb_pending_ops_reject: 0
vb_pending_ops_update: 0
vb_pending_perc_mem_resident: 0
vb_pending_queue_age: 0
vb_pending_queue_drain: 0
vb_pending_queue_fill: 0
vb_pending_queue_memory: 0
vb_pending_queue_pending: 0
vb_pending_queue_size: 0
vb_replica_curr_items: 350
vb_replica_eject: 0
vb_replica_expired: 0
vb_replica_ht_memory: 12849152
vb_replica_itm_memory: 113345
vb_replica_meta_data_memory: 23045
vb_replica_num: 512
vb_replica_num_non_resident: 0
vb_replica_ops_create: 497
vb_replica_ops_delete: 147
vb_replica_ops_reject: 0
vb_replica_ops_update: 0
vb_replica_perc_mem_resident: 100
vb_replica_queue_age: 14490000
vb_replica_queue_drain: 647
vb_replica_queue_fill: 649
vb_replica_queue_memory: 144
vb_replica_queue_pending: 974
vb_replica_queue_size: 2
version: 3.0.0-585-rel

Attaching cbcollect info