memcached crash during YCSB workload A

bin/ycsb load couchbase2 -P workloads/workloada -p writeallfields=true -threads 120 -p couchbase.host=172.23.132.15 -p couchbase.bucket=bucket-1 -p couchbase.password=password -p couchbase.upsert=true -p couchbase.boost=48 -p couchbase.epoll=true -p exportfile=ycsb_load_0.log  -p recordcount=10000000  2>ycsb_load_0_stderr.log

bin/ycsb run couchbase2 -P workloads/workloada -p writeallfields=true -threads 160 -p couchbase.host=172.23.132.15 -p couchbase.bucket=bucket-1 -p couchbase.password=password -p couchbase.upsert=true -p couchbase.boost=48 -p couchbase.epoll=true -p exportfile=ycsb_run_0.log  -p operationcount=1000000000  -p maxexecutiontime=600  -p recordcount=10000000  2>ycsb_run_0_stderr.log

2017-11-05T04:43:10.417579-08:00 FATAL *** Fatal error encountered during exception handling ***

2017-11-05T04:43:10.426499-08:00 FATAL Caught unhandled std::exception-derived exception. what(): Trying to set incorrect begin -1 on SeqRange: [665290, 665296]

2017-11-05T04:43:11.067949-08:00 FATAL Breakpad caught crash in memcached version 5.1.0-1374. Writing crash dump to /opt/couchbase/var/lib/couchbase/crash/48869bde-5a76-0a66-3ace7483-45a43c4d.dmp before terminating.

2017-11-05T04:43:11.068492-08:00 FATAL Stack backtrace of crashed thread:

2017-11-05T04:43:11.079777-08:00 FATAL     /opt/couchbase/bin/memcached() [0x400000+0x39fc5]

2017-11-05T04:43:11.079829-08:00 FATAL     /opt/couchbase/bin/memcached(_ZN15google_breakpad16ExceptionHandler12GenerateDumpEPNS0_12CrashContextE+0x3d4) [0x400000+0x99a54]

2017-11-05T04:43:11.079850-08:00 FATAL     /opt/couchbase/bin/memcached(_ZN15google_breakpad16ExceptionHandler13SignalHandlerEiP9siginfo_tPv+0x97) [0x400000+0x99d67]

2017-11-05T04:43:11.079872-08:00 FATAL     /lib64/libpthread.so.0() [0x7fa2d1dce000+0xf5e0]

2017-11-05T04:43:11.079922-08:00 FATAL     /lib64/libc.so.6(gsignal+0x37) [0x7fa2cfdff000+0x351f7]

2017-11-05T04:43:11.079967-08:00 FATAL     /lib64/libc.so.6(abort+0x148) [0x7fa2cfdff000+0x368e8]

2017-11-05T04:43:11.080030-08:00 FATAL     /lib64/libstdc++.so.6(_ZN9__gnu_cxx27__verbose_terminate_handlerEv+0x165) [0x7fa2d06da000+0x60ac5]

2017-11-05T04:43:11.080052-08:00 FATAL     /opt/couchbase/bin/memcached() [0x400000+0x3b39a]

2017-11-05T04:43:11.080116-08:00 FATAL     /lib64/libstdc++.so.6() [0x7fa2d06da000+0x5ea36]

2017-11-05T04:43:11.080175-08:00 FATAL     /lib64/libstdc++.so.6() [0x7fa2d06da000+0x5ea63]

2017-11-05T04:43:11.080231-08:00 FATAL     /lib64/libstdc++.so.6() [0x7fa2d06da000+0x5ec83]

2017-11-05T04:43:11.080255-08:00 FATAL     /opt/couchbase/bin/../lib/../lib/ep.so() [0x7fa2cc028000+0xe9a03]

2017-11-05T04:43:11.080275-08:00 FATAL     /opt/couchbase/bin/../lib/../lib/ep.so() [0x7fa2cc028000+0x9db96]

2017-11-05T04:43:11.080293-08:00 FATAL     /opt/couchbase/bin/../lib/../lib/ep.so() [0x7fa2cc028000+0x9cba7]

2017-11-05T04:43:11.080310-08:00 FATAL     /opt/couchbase/bin/../lib/../lib/ep.so() [0x7fa2cc028000+0xcc0e9]

2017-11-05T04:43:11.080327-08:00 FATAL     /opt/couchbase/bin/../lib/../lib/ep.so() [0x7fa2cc028000+0x9bee9]

2017-11-05T04:43:11.080344-08:00 FATAL     /opt/couchbase/bin/../lib/../lib/ep.so() [0x7fa2cc028000+0xa9bd1]

2017-11-05T04:43:11.080363-08:00 FATAL     /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0() [0x7fa2d240b000+0x702e]

2017-11-05T04:43:11.080386-08:00 FATAL     /lib64/libpthread.so.0() [0x7fa2d1dce000+0x7e25]

2017-11-05T04:43:11.080486-08:00 FATAL     /lib64/libc.so.6(clone+0x6d) [0x7fa2cfdff000+0xf834d]

#0  0x00007fa2cfe341f7 in __GI_raise (sig=sig@entry=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:56

56	  return INLINE_SYSCALL (tgkill, 3, pid, selftid, sig);

(gdb) bt full

#0  0x00007fa2cfe341f7 in __GI_raise (sig=sig@entry=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:56

        resultvar = 0

        pid = 620

        selftid = 810

#1  0x00007fa2cfe358e8 in __GI_abort () at abort.c:90

        save_stage = 2

        act = {__sigaction_handler = {sa_handler = 0x43204c4154414620, sa_sigaction = 0x43204c4154414620}, sa_mask = {__val = {7959303600787125601, 3762249738409226554, 3472339325703502137, 140337252909504, 

              140337252909504, 1, 140337252909635, 18446744073709551615, 140337249477660, 1, 140337252909504, 140335606871520, 140337249945693, 1, 140335774706736, 140335103609104}}, sa_flags = 10, 

          sa_restorer = 0x7fa26dff3700}

        sigs = {__val = {32, 0 <repeats 15 times>}}

#2  0x00007fa2d073aac5 in __gnu_cxx::__verbose_terminate_handler () at ../../../../libstdc++-v3/libsupc++/vterminate.cc:95

        terminating = false

        t = <optimized out>

#3  0x000000000043b39a in backtrace_terminate_handler () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:103

No locals.

#4  0x00007fa2d0738a36 in __cxxabiv1::__terminate (handler=<optimized out>) at ../../../../libstdc++-v3/libsupc++/eh_terminate.cc:38

No locals.

#5  0x00007fa2d0738a63 in std::terminate () at ../../../../libstdc++-v3/libsupc++/eh_terminate.cc:48

No locals.

#6  0x00007fa2d0738c83 in __cxxabiv1::__cxa_throw (obj=0x7fa250000980, tinfo=0x6d9660 <typeinfo for std::invalid_argument>, dest=0x4123f0 <_ZNSt16invalid_argumentD1Ev@plt>)

    at ../../../../libstdc++-v3/libsupc++/eh_throw.cc:87

        globals = <optimized out>

#7  0x00007fa2cc111a03 in setBegin (start=-1, this=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/linked_list.h:97

No locals.

#8  BasicLinkedList::purgeTombstones(long, std::function<bool ()>) (this=0x7fa2ce2c0e40, purgeUpToSeqno=<optimized out>, shouldPause=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/linked_list.cc:307

        rangeGuard = {_M_device = @0x7fa2ce2c0e98}

        it = <optimized out>

        rrGuard = {_M_device = 0x7fa2ce2c0ea0, _M_owns = true}

        startIt = <optimized out>

        purgedCount = 1

#9  0x00007fa2cc0c5b96 in EphemeralVBucket::purgeStaleItems(std::function<bool ()>) (this=this@entry=0x7fa2b82a3c00, shouldPauseCbk=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ephemeral_vb.cc:320

        seqListPurged = <optimized out>

#10 0x00007fa2cc0c4ba7 in EphemeralVBucket::StaleItemDeleter::visit (this=0x7fa1a9efef40, vb=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ephemeral_tombstone_purger.cc:199

        vbucket = 0x7fa2b82a3c00

#11 0x00007fa2cc0f40e9 in KVBucket::pauseResumeVisit (this=0x7fa2b909d800, visitor=..., start_pos=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kv_bucket.cc:2046

        paused = <optimized out>

        vb = <error reading variable: Cannot access memory at address 0x7fa2b820db68>

        vbid = 950

#12 0x00007fa2cc0c3ee9 in EphTombstoneStaleItemDeleter::run (this=0x7fa2b9037818) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ephemeral_tombstone_purger.cc:258

        start = <optimized out>

        end = <optimized out>

        duration_ms = <optimized out>

#13 0x00007fa2cc0d1bd1 in ExecutorThread::run (this=0x7fa2b854f000) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/executorthread.cc:122

        startReltime = 440

        woketime = <optimized out>

        again = <optimized out>

        runtime = <optimized out>

        q = <optimized out>

        tick = <optimized out>

#14 0x00007fa2d241202e in run (this=0x7fa2b85675a0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:59

No locals.

#15 platform_thread_wrap (arg=0x7fa2b85675a0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:72

        context = std::unique_ptr<CouchbaseThread> containing 0x7fa2b85675a0

#16 0x00007fa2d1dd5e25 in start_thread (arg=0x7fa26dff3700) at pthread_create.c:308

        __res = <optimized out>

        pd = 0x7fa26dff3700

        now = <optimized out>

        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {140335606871808, -5162969465144655788, 0, 140335606872512, 140335606871808, 140336854003712, 5124438172639248468, 5124200389801361492}, mask_was_saved = 0}}, 

          priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}

        not_first_call = <optimized out>

        pagesize_m1 = <optimized out>

        sp = <optimized out>

        freesize = <optimized out>

#17 0x00007fa2cfef734d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113