[4.x CLONE] - mcd abort during rebalance

Service 'memcached' exited with status 139. Restarting. Messages:     /opt/couchbase/b

terminate called after throwing an instance of 'std::logic_error'

  what():  CheckpointManager::queueDirty: lastBySeqno not in snapshot range. vb:1009 state:active snapshotStart:102954 lastBySeqno:102955 snapshotEnd:102954 genSeqno:False

2016-01-29T08:28:23.074101-08:00 WARNING Breakpad caught crash in memcached. Writing crash dump to /opt/couchbase/var/lib/couchbase/crash/385e6fb4-4e53-c4b0-19ed06db-2bcdb402.dmp before terminating.

2016-01-29T08:28:23.074147-08:00 WARNING Stack backtrace of crashed thread:

Thread 1 (LWP 5712):

#0  0x00007fe15f8cacc9 in __GI_raise (sig=sig@entry=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:56

#1  0x00007fe15f8ce0d8 in __GI_abort () at abort.c:89

#2  0x00007fe15fecf535 in __gnu_cxx::__verbose_terminate_handler() () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.19

#3  0x0000000000438b2a in backtrace_terminate_handler ()

    at /home/couchbase/serverjenkins/workspace/watson-unix/memcached/daemon/memcached.cc:2253

#4  0x74706f2f20202020 in ?? ()

#5  0x61626863756f632f in ?? ()

#6  0x6d2f6e69622f6573 in ?? ()

#7  0x6465686361636d65 in ?? ()

#8  0x333478305b202928 in ?? ()

#9  0x20200a5d34336261 in ?? ()

#10 0x632f74706f2f2020 in ?? ()

#11 0x657361626863756f in ?? ()

#12 0x2f2e2e2f6e69622f in ?? ()

#13 0x6562696c2f62696c in ?? ()

#14 0x726f635f746e6576 in ?? ()

#15 0x6f732e302e322d65 in ?? ()

#16 0x746e65766528352e in ?? ()

#17 0x6f6c5f657361625f in ?? ()

#18 0x38373678302b706f in ?? ()

#19 0x65663778305b2029 in ?? ()

#20 0x6336366564303631 in ?? ()

#21 0x2f202020200a5d38 in ?? ()

#22 0x63756f632f74706f in ?? ()

#23 0x69622f6573616268 in ?? ()

#24 0x6361636d656d2f6e in ?? ()

#25 0x305b202928646568 in ?? ()

#26 0x5d34613363343478 in ?? ()

#27 0x706f2f202020200a in ?? ()

#28 0x626863756f632f74 in ?? ()

#29 0x2f6e69622f657361 in ?? ()

---Type <return> to continue, or q <return> to quit---

#30 0x6c2f62696c2f2e2e in ?? ()

#31 0x6f6674616c706269 in ?? ()

#32 0x2e302e6f732e6d72 in ?? ()

#33 0x305b202928302e31 in ?? ()

#34 0x3831363165663778 in ?? ()

#35 0x200a5d6133353536 in ?? ()

#36 0x2f62696c2f202020 in ?? ()

#37 0x6c2d34365f363878 in ?? ()

#38 0x756e672d78756e69 in ?? ()

#39 0x7268747062696c2f in ?? ()

#40 0x302e6f732e646165 in ?? ()

#41 0x663778305b202928 in ?? ()

#42 0x3162373130363165 in ?? ()

#43 0x202020200a5d3238 in ?? ()

#44 0x3638782f62696c2f in ?? ()

#45 0x756e696c2d34365f in ?? ()

#46 0x696c2f756e672d78 in ?? ()

#47 0x28362e6f732e6362 in ?? ()

#48 0x78302b656e6f6c63 in ?? ()

#49 0x3778305b20296436 in ?? ()

#50 0x6538396635316566 in ?? ()

#51 0x0000000a5d643734 in ?? ()

#52 0x00007fe15c0f5f43 in memcpy (__len=<optimized out>, __src=<optimized out>, __dest=0x7fe112144d56)

    at /usr/include/x86_64-linux-gnu/bits/string3.h:51

#53 StoredValueFactory::newStoredValue (this=<optimized out>, itm=..., n=0x7fe15c1a12df, ht=..., setDirty=<optimized out>)

    at /home/couchbase/serverjenkins/workspace/watson-unix/ep-engine/src/stored-value.h:778

#54 0x00007fe15c0f62c5 in HashTable::unlocked_set (this=0x7fe09aae9660, v=@0x100000000: <error reading variable>, val=..., 

    cas=<optimized out>, allowExisting=<optimized out>, hasMetaData=<optimized out>, policy=FULL_EVICTION, nru=0 '\000', 

    maybeKeyExists=false, isReplication=true)

    at /home/couchbase/serverjenkins/workspace/watson-unix/ep-engine/src/stored-value.h:1123

#55 0x00007fe15c0eead5 in EventuallyPersistentStore::setWithMeta (this=<optimized out>, itm=..., cas=cas@entry=0, 

    seqno=seqno@entry=0x0, cookie=<optimized out>, force=force@entry=true, allowExisting=allowExisting@entry=true, 

    nru=nru@entry=2 '\002', genBySeqno=genBySeqno@entry=false, emd=0x7fe09b6bcee0, isReplication=isReplication@entry=true)

    at /home/couchbase/serverjenkins/workspace/watson-unix/ep-engine/src/ep.cc:2356

#56 0x00007fe15c0dbf24 in PassiveStream::processMutation (this=this@entry=0x7fe123646280, mutation=mutation@entry=0x7fe09b2ea820)

    at /home/couchbase/serverjenkins/workspace/watson-unix/ep-engine/src/dcp/stream.cc:1490

#57 0x00007fe15c0dc87b in PassiveStream::messageReceived (this=0x7fe123646280, resp=resp@entry=0x7fe09b2ea820)

    at /home/couchbase/serverjenkins/workspace/watson-unix/ep-engine/src/dcp/stream.cc:1351

#58 0x00007fe15c0c6d7f in DcpConsumer::mutation (this=0x7fe12397e200, opaque=<optimized out>, key=<optimized out>, nkey=20, 

    value=<optimized out>, nvalue=2585, cas=1454084903182532608, vbucket=1009, flags=2, datatype=1 '\001', locktime=0, 

    bySeqno=102955, revSeqno=28309, exptime=0, nru=2 '\002', meta=0x7fe1237bbc2a, nmeta=5)

    at /home/couchbase/serverjenkins/workspace/watson-unix/ep-engine/src/dcp/consumer.cc:291

#59 0x00007fe15c0fc8ea in EvpDcpMutation (handle=<optimized out>, cookie=0x7fe12390d000, opaque=208, key=0x7fe1237bb1fd, 

    nkey=<optimized out>, value=0x7fe1237bb211, nvalue=2585, cas=1454084903182532608, vbucket=1009, flags=2, datatype=1 '\001', 

    bySeqno=102955, revSeqno=28309, expiration=0, lockTime=0, meta=0x7fe1237bbc2a, nmeta=5, nru=2 '\002')

    at /home/couchbase/serverjenkins/workspace/watson-unix/ep-engine/src/ep_engine.cc:1612

#60 0x0000000000428d3a in dcp_mutation_executor (c=c@entry=0x7fe12390d000, packet=packet@entry=0x7fe1237bb1c6)

    at /home/couchbase/serverjenkins/workspace/watson-unix/memcached/daemon/mcbp_executors.cc:2387

#61 0x000000000043078a in process_bin_packet (c=0x7fe12390d000)

    at /home/couchbase/serverjenkins/workspace/watson-unix/memcached/daemon/mcbp_executors.cc:4705

#62 mcbp_complete_nread (c=c@entry=0x7fe12390d000)

    at /home/couchbase/serverjenkins/workspace/watson-unix/memcached/daemon/mcbp_executors.cc:4804

#63 0x0000000000446bac in conn_nread (c=0x7fe12390d000)

---Type <return> to continue, or q <return> to quit---

    at /home/couchbase/serverjenkins/workspace/watson-unix/memcached/daemon/statemachine_mcbp.cc:305

#64 0x000000000041e308 in execute (connection=..., this=<optimized out>)

    at /home/couchbase/serverjenkins/workspace/watson-unix/memcached/daemon/statemachine_mcbp.h:43

#65 McbpConnection::runStateMachinery (this=this@entry=0x7fe12390d000)

    at /home/couchbase/serverjenkins/workspace/watson-unix/memcached/daemon/connection_mcbp.cc:1000

#66 0x000000000041e34b in McbpConnection::runEventLoop (this=0x7fe12390d000, which=<optimized out>)

    at /home/couchbase/serverjenkins/workspace/watson-unix/memcached/daemon/connection_mcbp.cc:1266

#67 0x0000000000420970 in run_event_loop (c=0x7fe12390d000, which=<optimized out>)

    at /home/couchbase/serverjenkins/workspace/watson-unix/memcached/daemon/connections.cc:143

#68 0x000000000043ab34 in event_handler (fd=<optimized out>, which=<optimized out>, arg=0x7fe12390d000)

    at /home/couchbase/serverjenkins/workspace/watson-unix/memcached/daemon/memcached.cc:812

#69 0x00007fe160de66c8 in event_persist_closure (ev=<optimized out>, base=0x7fe15ea17300)

    at /home/couchbase/jenkins/workspace/cbdeps-build/label/ubuntu14.04/release/sherlock/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1319

#70 event_process_active_single_queue (activeq=0x7fe15ead4330, base=0x7fe15ea17300)

    at /home/couchbase/jenkins/workspace/cbdeps-build/label/ubuntu14.04/release/sherlock/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1363

#71 event_process_active (base=<optimized out>)

    at /home/couchbase/jenkins/workspace/cbdeps-build/label/ubuntu14.04/release/sherlock/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1438

#72 event_base_loop (base=0x7fe15ea17300, flags=0)

    at /home/couchbase/jenkins/workspace/cbdeps-build/label/ubuntu14.04/release/sherlock/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1639

#73 0x000000000044c3a4 in worker_libevent (arg=0x7fe158434fd8)

    at /home/couchbase/serverjenkins/workspace/watson-unix/memcached/daemon/thread.cc:231

#74 0x00007fe16186553a in run (this=0x7fe15ea7dfa0)

    at /home/couchbase/serverjenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54

#75 platform_thread_wrap (arg=0x7fe15ea7dfa0)

    at /home/couchbase/serverjenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66

#76 0x00007fe16017b182 in start_thread (arg=0x7fe151bf2700) at pthread_create.c:312

#77 0x00007fe15f98e47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

(gdb)