[system tests] memcached crash: Segmentation fault

4.5.0-1875

steps:
1. 3 nodes in cluster, 4 buckets. run data loader more then a day
2. setup replication from SRC to cluster DEST for all buckets.
3. add 172.23.106.125 node to src cluster http://172.23.105.207/(without rebalance)

result:

[root@localhost tmp]# ls -la
total 27945640
drwxrwxrwt. 3 root root 4096 Mar 24 03:01 .
dr-xr-xr-x. 22 root root 4096 Nov 16 11:48 ..
rw------ 1 couchbase couchbase 341061632 Mar 24 02:41 core.goxdcr.18128
rw------ 1 couchbase couchbase 296161280 Mar 24 02:55 core.goxdcr.19733
rw------ 1 couchbase couchbase 210489344 Mar 24 02:27 core.goxdcr.23308
rw------ 1 couchbase couchbase 381476864 Mar 24 02:27 core.mc:executor.18029
rw------ 1 couchbase couchbase 366796800 Mar 24 02:27 core.mc:executor.18095
rw------ 1 couchbase couchbase 2182930432 Mar 24 02:34 core.mc:executor.18145
rw------ 1 couchbase couchbase 498917376 Mar 24 02:34 core.mc:executor.18559
rw------ 1 couchbase couchbase 398254080 Mar 24 02:34 core.mc:executor.18589
rw------ 1 couchbase couchbase 559734784 Mar 24 02:35 core.mc:executor.18655
rw------ 1 couchbase couchbase 693952512 Mar 24 02:35 core.mc:executor.18686
rw------ 1 couchbase couchbase 490528768 Mar 24 02:35 core.mc:executor.18715
rw------ 1 couchbase couchbase 494723072 Mar 24 02:36 core.mc:executor.18794
rw------ 1 couchbase couchbase 341630976 Mar 24 02:36 core.mc:executor.18823
rw------ 1 couchbase couchbase 534564864 Mar 24 02:36 core.mc:executor.18854
rw------ 1 couchbase couchbase 1369235456 Mar 24 02:37 core.mc:executor.18883
rw------ 1 couchbase couchbase 343728128 Mar 24 02:37 core.mc:executor.18952
rw------ 1 couchbase couchbase 400351232 Mar 24 02:37 core.mc:executor.18981
rw------ 1 couchbase couchbase 366796800 Mar 24 02:37 core.mc:executor.19010
rw------ 1 couchbase couchbase 800907264 Mar 24 02:38 core.mc:executor.19089
rw------ 1 couchbase couchbase 341630976 Mar 24 02:38 core.mc:executor.19121
rw------ 1 couchbase couchbase 507305984 Mar 24 02:38 core.mc:executor.19150
rw------ 1 couchbase couchbase 551346176 Mar 24 02:38 core.mc:executor.19179
rw------ 1 couchbase couchbase 972873728 Mar 24 02:39 core.mc:executor.19208
rw------ 1 couchbase couchbase 368893952 Mar 24 02:39 core.mc:executor.19274
rw------ 1 couchbase couchbase 444391424 Mar 24 02:39 core.mc:executor.19305
rw------ 1 couchbase couchbase 507305984 Mar 24 02:39 core.mc:executor.19334
rw------ 1 couchbase couchbase 341630976 Mar 24 02:39 core.mc:executor.19363
rw------ 1 couchbase couchbase 540860416 Mar 24 02:40 core.mc:executor.19392
rw------ 1 couchbase couchbase 914153472 Mar 24 02:40 core.mc:executor.19471
rw------ 1 couchbase couchbase 528277504 Mar 24 02:40 core.mc:executor.19502
rw------ 1 couchbase couchbase 157011968 Mar 24 02:40 core.mc:executor.19531
rw------ 1 couchbase couchbase 450682880 Mar 24 02:40 core.mc:executor.19545
rw------ 1 couchbase couchbase 515694592 Mar 24 02:40 core.mc:executor.19574
rw------ 1 couchbase couchbase 387768320 Mar 24 02:40 core.mc:executor.19603
rw------ 1 couchbase couchbase 639426560 Mar 24 02:41 core.mc:executor.19669
rw------ 1 couchbase couchbase 754769920 Mar 24 02:42 core.mc:executor.19700
rw------ 1 couchbase couchbase 2849824768 Mar 24 02:47 core.mc:executor.19800
rw------ 1 couchbase couchbase 614264832 Mar 24 02:47 core.mc:executor.20051
rw------ 1 couchbase couchbase 423419904 Mar 24 02:47 core.mc:executor.20081
rw------ 1 couchbase couchbase 1159520256 Mar 24 02:48 core.mc:executor.20160
rw------ 1 couchbase couchbase 819781632 Mar 24 02:48 core.mc:executor.20191
rw------ 1 couchbase couchbase 381476864 Mar 24 02:48 core.mc:executor.20220
rw------ 1 couchbase couchbase 1329389568 Mar 24 02:49 core.mc:executor.20249
rw------ 1 couchbase couchbase 421322752 Mar 24 02:49 core.mc:executor.20318
rw------ 1 couchbase couchbase 836558848 Mar 24 02:49 core.mc:executor.20347
rw------ 1 couchbase couchbase 465362944 Mar 24 02:49 core.mc:executor.20376
rw------ 1 couchbase couchbase 1639768064 Mar 24 02:50 core.mc:executor.20455
rw------ 1 couchbase couchbase 452780032 Mar 24 02:51 core.mc:executor.20486
rw------ 1 couchbase couchbase 215756800 Mar 24 02:51 core.mc:executor.20554
rw------ 1 couchbase couchbase 612163584 Mar 24 02:52 core.mc:executor.20573
rw------ 1 couchbase couchbase 1794957312 Mar 24 02:53 core.mc:executor.20602
rw------ 1 couchbase couchbase 1826414592 Mar 24 02:54 core.mc:executor.20887
rw------ 1 couchbase couchbase 572317696 Mar 24 02:54 core.mc:executor.21209
rw------ 1 couchbase couchbase 454877184 Mar 24 02:55 core.mc:executor.21238
rw------ 1 couchbase couchbase 574414848 Mar 24 02:56 core.mc:executor.21306
rw------ 1 couchbase couchbase 712826880 Mar 24 02:56 core.mc:executor.21354
rw------ 1 couchbase couchbase 798810112 Mar 24 02:56 core.mc:executor.21468
rw------ 1 couchbase couchbase 1522327552 Mar 24 02:57 core.mc:executor.21632
rw------ 1 couchbase couchbase 1316806656 Mar 24 02:57 core.mc:executor.22116
rw------ 1 couchbase couchbase 415031296 Mar 24 02:57 core.mc:executor.22558
rw------ 1 couchbase couchbase 2940006400 Mar 24 02:26 core.mc:executor.22883
rw------ 1 couchbase couchbase 433905664 Mar 24 02:58 core.mc:executor.23121
rw------ 1 couchbase couchbase 555573248 Mar 24 02:58 core.mc:executor.23575
rw------ 1 couchbase couchbase 5287936 Mar 24 02:59 core.mc:executor.25276
rw------ 1 couchbase couchbase 998256640 Mar 24 03:00 core.mc:executor.25669
rw------ 1 couchbase couchbase 7544832 Mar 24 03:01 core.mc:executor.25767

[root@localhost tmp]# gdb core.mc:executor.18029
GNU gdb (GDB) Red Hat Enterprise Linux (7.2-83.el6)
Copyright (C) 2010 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Missing separate debuginfo for the main executable file
Try: yum --enablerepo='-debug' install /usr/lib/debug/.build-id/26/3b212f52cf570ecbe47b13758d6bba57178ff3
[New Thread 18041]
[New Thread 18047]
[New Thread 18029]
[New Thread 18034]
[New Thread 18044]
[New Thread 18048]
[New Thread 18049]
[New Thread 18037]
[New Thread 18050]
[New Thread 18030]
[New Thread 18051]
[New Thread 18052]
[New Thread 18038]
[New Thread 18053]
[New Thread 18054]
[New Thread 18055]
[New Thread 18031]
[New Thread 18039]
[New Thread 18056]
[New Thread 18035]
[New Thread 18032]
[New Thread 18036]
[New Thread 18033]
[New Thread 18040]
Core was generated by `/opt/couchbase/bin/memcached -C /opt/couchbase/var/lib/couchbase/config/memcach'.
Program terminated with signal 11, Segmentation fault.
#0 0x00007f3994f72525 in ?? ()
"/tmp/core.mc:executor.18029" is a core file.
Please specify an executable to debug.
(gdb) tb
Temporary breakpoint 1 at 0x7f3994f72525
(gdb) bt
#0 0x00007f3994f72525 in ?? ()
#1 0x000000000044a410 in ?? ()
#2 0x00007f39910a3100 in ?? ()
#3 0x0000000000438202 in ?? ()
#4 0x0000000000000000 in ?? ()
(gdb)
#0 0x00007f3994f72525 in ?? ()
#1 0x000000000044a410 in ?? ()
#2 0x00007f39910a3100 in ?? ()
#3 0x0000000000438202 in ?? ()
#4 0x0000000000000000 in ?? ()
(gdb)
#0 0x00007f3994f72525 in ?? ()
#1 0x000000000044a410 in ?? ()
#2 0x00007f39910a3100 in ?? ()
#3 0x0000000000438202 in ?? ()
#4 0x0000000000000000 in ?? ()
(gdb) ^C(gdb) Quit
(gdb) quit
[root@localhost tmp]# gdb /opt/couchbase/bin/memcached core.mc:executor.18029
GNU gdb (GDB) Red Hat Enterprise Linux (7.2-83.el6)
Copyright (C) 2010 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /opt/couchbase/bin/memcached...(no debugging symbols found)...done.
[New Thread 18041]
[New Thread 18047]
[New Thread 18029]
[New Thread 18034]
[New Thread 18044]
[New Thread 18048]
[New Thread 18049]
[New Thread 18037]
[New Thread 18050]
[New Thread 18030]
[New Thread 18051]
[New Thread 18052]
[New Thread 18038]
[New Thread 18053]
[New Thread 18054]
[New Thread 18055]
[New Thread 18031]
[New Thread 18039]
[New Thread 18056]
[New Thread 18035]
[New Thread 18032]
[New Thread 18036]
[New Thread 18033]
[New Thread 18040]
Reading symbols from /opt/couchbase/bin/../lib/libauditd.so.0.1.0...(no debugging symbols found)...done.
Loaded symbols for /opt/couchbase/bin/../lib/libauditd.so.0.1.0
Reading symbols from /opt/couchbase/bin/../lib/libmcd_time.so.1.0.0...(no debugging symbols found)...done.
Loaded symbols for /opt/couchbase/bin/../lib/libmcd_time.so.1.0.0
Reading symbols from /opt/couchbase/bin/../lib/libdirutils.so.0.1.0...(no debugging symbols found)...done.
Loaded symbols for /opt/couchbase/bin/../lib/libdirutils.so.0.1.0
Reading symbols from /opt/couchbase/bin/../lib/libmcd_util.so.1.0.0...(no debugging symbols found)...done.
Loaded symbols for /opt/couchbase/bin/../lib/libmcd_util.so.1.0.0
Reading symbols from /opt/couchbase/bin/../lib/libcbsasl.so.1.1.1...(no debugging symbols found)...done.
Loaded symbols for /opt/couchbase/bin/../lib/libcbsasl.so.1.1.1
Reading symbols from /opt/couchbase/bin/../lib/libcJSON.so.1.1.0...(no debugging symbols found)...done.
Loaded symbols for /opt/couchbase/bin/../lib/libcJSON.so.1.1.0
Reading symbols from /opt/couchbase/bin/../lib/libJSON_checker.so.1.0.0...(no debugging symbols found)...done.
Loaded symbols for /opt/couchbase/bin/../lib/libJSON_checker.so.1.0.0
Reading symbols from /opt/couchbase/bin/../lib/libplatform.so.0.1.0...(no debugging symbols found)...done.
Loaded symbols for /opt/couchbase/bin/../lib/libplatform.so.0.1.0
Reading symbols from /lib64/libdl.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libdl.so.2
Reading symbols from /lib64/libpthread.so.0...(no debugging symbols found)...done.
[Thread debugging using libthread_db enabled]
Loaded symbols for /lib64/libpthread.so.0
Reading symbols from /lib64/librt.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/librt.so.1
Reading symbols from /opt/couchbase/bin/../lib/libsnappy.so.1...(no debugging symbols found)...done.
Loaded symbols for /opt/couchbase/bin/../lib/libsnappy.so.1
Reading symbols from /opt/couchbase/bin/../lib/libjemalloc.so.2...(no debugging symbols found)...done.
Loaded symbols for /opt/couchbase/bin/../lib/libjemalloc.so.2
Reading symbols from /opt/couchbase/bin/../lib/libevent_core-2.0.so.5...(no debugging symbols found)...done.
Loaded symbols for /opt/couchbase/bin/../lib/libevent_core-2.0.so.5
Reading symbols from /usr/lib64/libssl.so.10...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libssl.so.10
Reading symbols from /usr/lib64/libcrypto.so.10...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libcrypto.so.10
Reading symbols from /opt/couchbase/bin/../lib/libnuma.so.1...(no debugging symbols found)...done.
Loaded symbols for /opt/couchbase/bin/../lib/libnuma.so.1
Reading symbols from /usr/lib64/libstdc++.so.6...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libstdc++.so.6
Reading symbols from /lib64/libm.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib64/libm.so.6
Reading symbols from /lib64/libgcc_s.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libgcc_s.so.1
Reading symbols from /lib64/libc.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib64/libc.so.6
Reading symbols from /lib64/ld-linux-x86-64.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/ld-linux-x86-64.so.2
Reading symbols from /lib64/libgssapi_krb5.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libgssapi_krb5.so.2
Reading symbols from /lib64/libkrb5.so.3...(no debugging symbols found)...done.
Loaded symbols for /lib64/libkrb5.so.3
Reading symbols from /lib64/libcom_err.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libcom_err.so.2
Reading symbols from /lib64/libk5crypto.so.3...(no debugging symbols found)...done.
Loaded symbols for /lib64/libk5crypto.so.3
Reading symbols from /lib64/libz.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libz.so.1
Reading symbols from /lib64/libkrb5support.so.0...(no debugging symbols found)...done.
Loaded symbols for /lib64/libkrb5support.so.0
Reading symbols from /lib64/libkeyutils.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libkeyutils.so.1
Reading symbols from /lib64/libresolv.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libresolv.so.2
Reading symbols from /lib64/libselinux.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libselinux.so.1
Reading symbols from /opt/couchbase/lib/memcached/stdin_term_handler.so...(no debugging symbols found)...done.
Loaded symbols for /opt/couchbase/lib/memcached/stdin_term_handler.so
Reading symbols from /opt/couchbase/lib/memcached/file_logger.so...(no debugging symbols found)...done.
Loaded symbols for /opt/couchbase/lib/memcached/file_logger.so
Reading symbols from /opt/couchbase/bin/../lib/../lib/nobucket.so...(no debugging symbols found)...done.
Loaded symbols for /opt/couchbase/bin/../lib/../lib/nobucket.so
Reading symbols from /opt/couchbase/bin/../lib/../lib/default_engine.so...(no debugging symbols found)...done.
Loaded symbols for /opt/couchbase/bin/../lib/../lib/default_engine.so
Reading symbols from /opt/couchbase/bin/../lib/../lib/ep.so...(no debugging symbols found)...done.
Loaded symbols for /opt/couchbase/bin/../lib/../lib/ep.so
Reading symbols from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so...(no debugging symbols found)...done.
Loaded symbols for /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
Reading symbols from /opt/couchbase/bin/../lib/../lib/../lib/libforestdb.so...(no debugging symbols found)...done.
Loaded symbols for /opt/couchbase/bin/../lib/../lib/../lib/libforestdb.so
Reading symbols from /opt/couchbase/bin/../lib/../lib/../lib/libv8.so...(no debugging symbols found)...done.
Loaded symbols for /opt/couchbase/bin/../lib/../lib/../lib/libv8.so
Reading symbols from /opt/couchbase/bin/../lib/../lib/../lib/libicuuc.so.54...(no debugging symbols found)...done.
Loaded symbols for /opt/couchbase/bin/../lib/../lib/../lib/libicuuc.so.54
Reading symbols from /opt/couchbase/bin/../lib/../lib/../lib/libicudata.so.54...(no debugging symbols found)...done.
Loaded symbols for /opt/couchbase/bin/../lib/../lib/../lib/libicudata.so.54
Reading symbols from /opt/couchbase/bin/../lib/../lib/../lib/libicui18n.so.54...(no debugging symbols found)...done.
Loaded symbols for /opt/couchbase/bin/../lib/../lib/../lib/libicui18n.so.54
Core was generated by `/opt/couchbase/bin/memcached -C /opt/couchbase/var/lib/couchbase/config/memcach'.
Program terminated with signal 11, Segmentation fault.
#0 0x00007f3994f72525 in cbsasl_server_step () from /opt/couchbase/bin/../lib/libcbsasl.so.1.1.1
Missing separate debuginfos, use: debuginfo-install couchbase-server-4.5.0-1875.x86_64
(gdb) t a a bt

Thread 24 (Thread 0x7f39851f8700 (LWP 18040)):
#0 0x00007f399452763c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007f39931f42fc in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib64/libstdc++.so.6
#2 0x00000000004382b0 in ?? ()
#3 0x00007f399494645b in Couchbase::Thread::thread_entry() () from /opt/couchbase/bin/../lib/libplatform.so.0.1.0
#4 0x00007f399494260a in ?? () from /opt/couchbase/bin/../lib/libplatform.so.0.1.0
#5 0x00007f3994523a51 in start_thread () from /lib64/libpthread.so.0
#6 0x00007f39929f893d in clone () from /lib64/libc.so.6

Thread 23 (Thread 0x7f39897ff700 (LWP 18033)):
#0 0x00007f39929f8f33 in epoll_wait () from /lib64/libc.so.6
#1 0x00007f3993cb9e93 in ?? () from /opt/couchbase/bin/../lib/libevent_core-2.0.so.5
#2 0x00007f3993ca6f42 in event_base_loop () from /opt/couchbase/bin/../lib/libevent_core-2.0.so.5
#3 0x0000000000420784 in ?? ()
#4 0x00007f399494260a in ?? () from /opt/couchbase/bin/../lib/libplatform.so.0.1.0
#5 0x00007f3994523a51 in start_thread () from /lib64/libpthread.so.0
#6 0x00007f39929f893d in clone () from /lib64/libc.so.6

Thread 22 (Thread 0x7f39879fc700 (LWP 18036)):
#0 0x00007f39929f8f33 in epoll_wait () from /lib64/libc.so.6
#1 0x00007f3993cb9e93 in ?? () from /opt/couchbase/bin/../lib/libevent_core-2.0.so.5
#2 0x00007f3993ca6f42 in event_base_loop () from /opt/couchbase/bin/../lib/libevent_core-2.0.so.5
#3 0x0000000000420784 in ?? ()
#4 0x00007f399494260a in ?? () from /opt/couchbase/bin/../lib/libplatform.so.0.1.0
#5 0x00007f3994523a51 in start_thread () from /lib64/libpthread.so.0
#6 0x00007f39929f893d in clone () from /lib64/libc.so.6

Thread 21 (Thread 0x7f398f1fe700 (LWP 18032)):
#0 0x00007f3994527a0e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007f3994942e09 in cb_cond_timedwait () from /opt/couchbase/bin/../lib/libplatform.so.0.1.0
#2 0x00007f3995793a9e in ?? () from /opt/couchbase/bin/../lib/libauditd.so.0.1.0
#3 0x00007f399494260a in ?? () from /opt/couchbase/bin/../lib/libplatform.so.0.1.0
#4 0x00007f3994523a51 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f39929f893d in clone () from /lib64/libc.so.6

Thread 20 (Thread 0x7f39883fd700 (LWP 18035)):
#0 0x00007f39929f8f33 in epoll_wait () from /lib64/libc.so.6
#1 0x00007f3993cb9e93 in ?? () from /opt/couchbase/bin/../lib/libevent_core-2.0.so.5
#2 0x00007f3993ca6f42 in event_base_loop () from /opt/couchbase/bin/../lib/libevent_core-2.0.so.5
#3 0x0000000000420784 in ?? ()
#4 0x00007f399494260a in ?? () from /opt/couchbase/bin/../lib/libplatform.so.0.1.0
#5 0x00007f3994523a51 in start_thread () from /lib64/libpthread.so.0
#6 0x00007f39929f893d in clone () from /lib64/libc.so.6

Thread 19 (Thread 0x7f397a9f6700 (LWP 18056)):
#0 0x00007f3994527a0e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007f398e1530ab in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#2 0x00007f398e155150 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#3 0x00007f398e11ab46 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#4 0x00007f398e11ac21 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#5 0x00007f398e1234e8 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#6 0x00007f399494260a in ?? () from /opt/couchbase/bin/../lib/libplatform.so.0.1.0
#7 0x00007f3994523a51 in start_thread () from /lib64/libpthread.so.0
#8 0x00007f39929f893d in clone () from /lib64/libc.so.6
--Type <return> to continue, or q <return> to quit--

Thread 18 (Thread 0x7f3985bf9700 (LWP 18039)):
#0 0x00007f399452763c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007f39931f42fc in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib64/libstdc++.so.6
#2 0x00000000004382b0 in ?? ()
#3 0x00007f399494645b in Couchbase::Thread::thread_entry() () from /opt/couchbase/bin/../lib/libplatform.so.0.1.0
#4 0x00007f399494260a in ?? () from /opt/couchbase/bin/../lib/libplatform.so.0.1.0
#5 0x00007f3994523a51 in start_thread () from /lib64/libpthread.so.0
#6 0x00007f39929f893d in clone () from /lib64/libc.so.6

Thread 17 (Thread 0x7f398fbff700 (LWP 18031)):
#0 0x00007f3994527a0e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007f3994942e09 in cb_cond_timedwait () from /opt/couchbase/bin/../lib/libplatform.so.0.1.0
#2 0x00007f39901f9a25 in ?? () from /opt/couchbase/lib/memcached/file_logger.so
#3 0x00007f399494260a in ?? () from /opt/couchbase/bin/../lib/libplatform.so.0.1.0
#4 0x00007f3994523a51 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f39929f893d in clone () from /lib64/libc.so.6

Thread 16 (Thread 0x7f397b3f7700 (LWP 18055)):
#0 0x00007f399452b0e3 in pread64 () from /lib64/libpthread.so.0
#1 0x00007f398de7b74a in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#2 0x00007f398e17faf7 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#3 0x00007f398de661bc in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#4 0x00007f398de61cb0 in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#5 0x00007f398de62095 in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#6 0x00007f398de60b25 in couchstore_open_doc_with_docinfo () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#7 0x00007f398e172056 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#8 0x00007f398de608ae in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#9 0x00007f398de5f359 in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#10 0x00007f398de5f1a7 in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#11 0x00007f398de5f1a7 in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#12 0x00007f398de60d77 in couchstore_changes_since () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#13 0x00007f398e16e5b7 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#14 0x00007f398e0c4f53 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#15 0x00007f398e0bfda9 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#16 0x00007f398e0c00bd in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#17 0x00007f398e12333e in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#18 0x00007f399494260a in ?? () from /opt/couchbase/bin/../lib/libplatform.so.0.1.0
#19 0x00007f3994523a51 in start_thread () from /lib64/libpthread.so.0
#20 0x00007f39929f893d in clone () from /lib64/libc.so.6

Thread 15 (Thread 0x7f397bdf8700 (LWP 18054)):
#0 0x00007f3994527a0e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007f398e1530ab in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#2 0x00007f398e155150 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#3 0x00007f398e11ab46 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#4 0x00007f398e11ac21 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#5 0x00007f398e1234e8 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#6 0x00007f399494260a in ?? () from /opt/couchbase/bin/../lib/libplatform.so.0.1.0
#7 0x00007f3994523a51 in start_thread () from /lib64/libpthread.so.0
#8 0x00007f39929f893d in clone () from /lib64/libc.so.6

Thread 14 (Thread 0x7f397c7f9700 (LWP 18053)):
#0 0x00007f3994527a0e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007f398e1530ab in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
--Type <return> to continue, or q <return> to quit--
#2 0x00007f398e155150 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#3 0x00007f398e11ab46 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#4 0x00007f398e11ac21 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#5 0x00007f398e1234e8 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#6 0x00007f399494260a in ?? () from /opt/couchbase/bin/../lib/libplatform.so.0.1.0
#7 0x00007f3994523a51 in start_thread () from /lib64/libpthread.so.0
#8 0x00007f39929f893d in clone () from /lib64/libc.so.6

Thread 13 (Thread 0x7f39865fa700 (LWP 18038)):
#0 0x00007f399452763c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007f39931f42fc in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib64/libstdc++.so.6
#2 0x00000000004382b0 in ?? ()
#3 0x00007f399494645b in Couchbase::Thread::thread_entry() () from /opt/couchbase/bin/../lib/libplatform.so.0.1.0
#4 0x00007f399494260a in ?? () from /opt/couchbase/bin/../lib/libplatform.so.0.1.0
#5 0x00007f3994523a51 in start_thread () from /lib64/libpthread.so.0
#6 0x00007f39929f893d in clone () from /lib64/libc.so.6

Thread 12 (Thread 0x7f397d1fa700 (LWP 18052)):
#0 0x00007f3994527a0e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007f398e1530ab in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#2 0x00007f398e155150 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#3 0x00007f398e11ab46 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#4 0x00007f398e11ac21 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#5 0x00007f398e1234e8 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#6 0x00007f399494260a in ?? () from /opt/couchbase/bin/../lib/libplatform.so.0.1.0
#7 0x00007f3994523a51 in start_thread () from /lib64/libpthread.so.0
#8 0x00007f39929f893d in clone () from /lib64/libc.so.6

Thread 11 (Thread 0x7f397dbfb700 (LWP 18051)):
#0 0x00007f3994527a0e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007f398e1530ab in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#2 0x00007f398e155150 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#3 0x00007f398e11ab46 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#4 0x00007f398e11ac21 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#5 0x00007f398e1234e8 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#6 0x00007f399494260a in ?? () from /opt/couchbase/bin/../lib/libplatform.so.0.1.0
#7 0x00007f3994523a51 in start_thread () from /lib64/libpthread.so.0
#8 0x00007f39929f893d in clone () from /lib64/libc.so.6

Thread 10 (Thread 0x7f3990dfd700 (LWP 18030)):
#0 0x00007f39929ef113 in poll () from /lib64/libc.so.6

#1 0x00007f3990dfeb48 in ?? () from /opt/couchbase/lib/memcached/stdin_term_handler.so
#2 0x00007f399494260a in ?? () from /opt/couchbase/bin/../lib/libplatform.so.0.1.0
#3 0x00007f3994523a51 in start_thread () from /lib64/libpthread.so.0
#4 0x00007f39929f893d in clone () from /lib64/libc.so.6

Thread 9 (Thread 0x7f397e5fc700 (LWP 18050)):
#0 0x00007f399452b0e3 in pread64 () from /lib64/libpthread.so.0
#1 0x00007f398de7b74a in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#2 0x00007f398e17faf7 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#3 0x00007f398de661bc in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#4 0x00007f398de61cb0 in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#5 0x00007f398de62095 in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#6 0x00007f398de5f046 in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#7 0x00007f398de5f1a7 in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
--Type <return> to continue, or q <return> to quit--
#8 0x00007f398de5f1a7 in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#9 0x00007f398de60d77 in couchstore_changes_since () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#10 0x00007f398e16e5b7 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#11 0x00007f398e1653ab in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#12 0x00007f398e16a571 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#13 0x00007f398e12333e in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#14 0x00007f399494260a in ?? () from /opt/couchbase/bin/../lib/libplatform.so.0.1.0
#15 0x00007f3994523a51 in start_thread () from /lib64/libpthread.so.0
#16 0x00007f39929f893d in clone () from /lib64/libc.so.6

Thread 8 (Thread 0x7f3986ffb700 (LWP 18037)):
#0 0x00007f39929f8f33 in epoll_wait () from /lib64/libc.so.6

#1 0x00007f3993cb9e93 in ?? () from /opt/couchbase/bin/../lib/libevent_core-2.0.so.5
#2 0x00007f3993ca6f42 in event_base_loop () from /opt/couchbase/bin/../lib/libevent_core-2.0.so.5
#3 0x0000000000420784 in ?? ()
#4 0x00007f399494260a in ?? () from /opt/couchbase/bin/../lib/libplatform.so.0.1.0
#5 0x00007f3994523a51 in start_thread () from /lib64/libpthread.so.0
#6 0x00007f39929f893d in clone () from /lib64/libc.so.6

Thread 7 (Thread 0x7f397effd700 (LWP 18049)):
#0 0x00007f399452b0e3 in pread64 () from /lib64/libpthread.so.0
#1 0x00007f398de7b74a in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#2 0x00007f398e17faf7 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#3 0x00007f398de661bc in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#4 0x00007f398de61cb0 in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#5 0x00007f398de62095 in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#6 0x00007f398de5f046 in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#7 0x00007f398de5f1a7 in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#8 0x00007f398de5f1a7 in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#9 0x00007f398de60d77 in couchstore_changes_since () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#10 0x00007f398e16e5b7 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#11 0x00007f398e1653ab in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#12 0x00007f398e16a571 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#13 0x00007f398e12333e in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#14 0x00007f399494260a in ?? () from /opt/couchbase/bin/../lib/libplatform.so.0.1.0
#15 0x00007f3994523a51 in start_thread () from /lib64/libpthread.so.0
#16 0x00007f39929f893d in clone () from /lib64/libc.so.6

Thread 6 (Thread 0x7f397f9fe700 (LWP 18048)):
#0 0x00007f399452b0e3 in pread64 () from /lib64/libpthread.so.0

#1 0x00007f398de7b74a in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#2 0x00007f398e17faf7 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#3 0x00007f398de661bc in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#4 0x00007f398de61cb0 in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#5 0x00007f398de62095 in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#6 0x00007f398de5f046 in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#7 0x00007f398de5f1a7 in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#8 0x00007f398de60d77 in couchstore_changes_since () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#9 0x00007f398e16e5b7 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#10 0x00007f398e1653ab in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#11 0x00007f398e16a571 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#12 0x00007f398e12333e in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#13 0x00007f399494260a in ?? () from /opt/couchbase/bin/../lib/libplatform.so.0.1.0
#14 0x00007f3994523a51 in start_thread () from /lib64/libpthread.so.0
#15 0x00007f39929f893d in clone () from /lib64/libc.so.6
--Type <return> to continue, or q <return> to quit--

Thread 5 (Thread 0x7f39829f4700 (LWP 18044)):
#0 0x00007f3994527a0e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007f398e12ed90 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#2 0x00007f399494260a in ?? () from /opt/couchbase/bin/../lib/libplatform.so.0.1.0
#3 0x00007f3994523a51 in start_thread () from /lib64/libpthread.so.0
#4 0x00007f39929f893d in clone () from /lib64/libc.so.6

Thread 4 (Thread 0x7f3988dfe700 (LWP 18034)):
#0 0x00007f399452adcd in sendmsg () from /lib64/libpthread.so.0
#1 0x000000000042fd32 in ?? ()
#2 0x000000000042fac9 in ?? ()
#3 0x000000000044bf92 in ?? ()
#4 0x0000000000430aa8 in ?? ()
#5 0x0000000000430aeb in ?? ()
#6 0x0000000000434080 in ?? ()
#7 0x0000000000410e41 in _start ()

Thread 3 (Thread 0x7f3995ba8820 (LWP 18029)):
#0 0x00007f39929f8f33 in epoll_wait () from /lib64/libc.so.6

#1 0x00007f3993cb9e93 in ?? () from /opt/couchbase/bin/../lib/libevent_core-2.0.so.5
#2 0x00007f3993ca6f42 in event_base_loop () from /opt/couchbase/bin/../lib/libevent_core-2.0.so.5
#3 0x00000000004148b9 in _start ()

Thread 2 (Thread 0x7f39803ff700 (LWP 18047)):
#0 0x00007f399452b0e3 in pread64 () from /lib64/libpthread.so.0
#1 0x00007f398de7b74a in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#2 0x00007f398e17faf7 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#3 0x00007f398de661bc in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#4 0x00007f398de61cb0 in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#5 0x00007f398de62095 in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#6 0x00007f398de5f046 in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#7 0x00007f398de5f1a7 in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#8 0x00007f398de5f1a7 in ?? () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#9 0x00007f398de60d77 in couchstore_changes_since () from /opt/couchbase/bin/../lib/../lib/../lib/libcouchstore.so
#10 0x00007f398e16e5b7 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#11 0x00007f398e1653ab in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#12 0x00007f398e16a571 in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#13 0x00007f398e12333e in ?? () from /opt/couchbase/bin/../lib/../lib/ep.so
#14 0x00007f399494260a in ?? () from /opt/couchbase/bin/../lib/libplatform.so.0.1.0
#15 0x00007f3994523a51 in start_thread () from /lib64/libpthread.so.0
#16 0x00007f39929f893d in clone () from /lib64/libc.so.6

Thread 1 (Thread 0x7f39847f7700 (LWP 18041)):
#0 0x00007f3994f72525 in cbsasl_server_step () from /opt/couchbase/bin/../lib/libcbsasl.so.1.1.1
#1 0x000000000044a410 in ?? ()
#2 0x0000000000438202 in ?? ()
#3 0x00007f399494645b in Couchbase::Thread::thread_entry() () from /opt/couchbase/bin/../lib/libplatform.so.0.1.0
#4 0x00007f399494260a in ?? () from /opt/couchbase/bin/../lib/libplatform.so.0.1.0
#5 0x00007f3994523a51 in start_thread () from /lib64/libpthread.so.0
#6 0x00007f39929f893d in clone () from /lib64/libc.so.6