Details
-
Bug
-
Resolution: Fixed
-
Critical
-
4.1.1, 4.5.0
-
Security Level: Public
-
Untriaged
-
Unknown
Description
Loaded 2 nodes to ~50% dgm + xdcr to 1 remote node
hard failover 1 node
rebalance node back in failed with...
Service 'memcached' exited with status 139. Restarting. Messages: /opt/couchbase/b
|
terminate called after throwing an instance of 'std::logic_error'
|
what(): CheckpointManager::queueDirty: lastBySeqno not in snapshot range. vb:1009 state:active snapshotStart:102954 lastBySeqno:102955 snapshotEnd:102954 genSeqno:False
|
2016-01-29T08:28:23.074101-08:00 WARNING Breakpad caught crash in memcached. Writing crash dump to /opt/couchbase/var/lib/couchbase/crash/385e6fb4-4e53-c4b0-19ed06db-2bcdb402.dmp before terminating.
|
2016-01-29T08:28:23.074147-08:00 WARNING Stack backtrace of crashed thread:
|
|
|
Thread 1 (LWP 5712):
|
#0 0x00007fe15f8cacc9 in __GI_raise (sig=sig@entry=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:56
|
#1 0x00007fe15f8ce0d8 in __GI_abort () at abort.c:89
|
#2 0x00007fe15fecf535 in __gnu_cxx::__verbose_terminate_handler() () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.19
|
#3 0x0000000000438b2a in backtrace_terminate_handler ()
|
at /home/couchbase/serverjenkins/workspace/watson-unix/memcached/daemon/memcached.cc:2253
|
#4 0x74706f2f20202020 in ?? ()
|
#5 0x61626863756f632f in ?? ()
|
#6 0x6d2f6e69622f6573 in ?? ()
|
#7 0x6465686361636d65 in ?? ()
|
#8 0x333478305b202928 in ?? ()
|
#9 0x20200a5d34336261 in ?? ()
|
#10 0x632f74706f2f2020 in ?? ()
|
#11 0x657361626863756f in ?? ()
|
#12 0x2f2e2e2f6e69622f in ?? ()
|
#13 0x6562696c2f62696c in ?? ()
|
#14 0x726f635f746e6576 in ?? ()
|
#15 0x6f732e302e322d65 in ?? ()
|
#16 0x746e65766528352e in ?? ()
|
#17 0x6f6c5f657361625f in ?? ()
|
#18 0x38373678302b706f in ?? ()
|
#19 0x65663778305b2029 in ?? ()
|
#20 0x6336366564303631 in ?? ()
|
#21 0x2f202020200a5d38 in ?? ()
|
#22 0x63756f632f74706f in ?? ()
|
#23 0x69622f6573616268 in ?? ()
|
#24 0x6361636d656d2f6e in ?? ()
|
#25 0x305b202928646568 in ?? ()
|
#26 0x5d34613363343478 in ?? ()
|
#27 0x706f2f202020200a in ?? ()
|
#28 0x626863756f632f74 in ?? ()
|
#29 0x2f6e69622f657361 in ?? ()
|
---Type <return> to continue, or q <return> to quit---
|
#30 0x6c2f62696c2f2e2e in ?? ()
|
#31 0x6f6674616c706269 in ?? ()
|
#32 0x2e302e6f732e6d72 in ?? ()
|
#33 0x305b202928302e31 in ?? ()
|
#34 0x3831363165663778 in ?? ()
|
#35 0x200a5d6133353536 in ?? ()
|
#36 0x2f62696c2f202020 in ?? ()
|
#37 0x6c2d34365f363878 in ?? ()
|
#38 0x756e672d78756e69 in ?? ()
|
#39 0x7268747062696c2f in ?? ()
|
#40 0x302e6f732e646165 in ?? ()
|
#41 0x663778305b202928 in ?? ()
|
#42 0x3162373130363165 in ?? ()
|
#43 0x202020200a5d3238 in ?? ()
|
#44 0x3638782f62696c2f in ?? ()
|
#45 0x756e696c2d34365f in ?? ()
|
#46 0x696c2f756e672d78 in ?? ()
|
#47 0x28362e6f732e6362 in ?? ()
|
#48 0x78302b656e6f6c63 in ?? ()
|
#49 0x3778305b20296436 in ?? ()
|
#50 0x6538396635316566 in ?? ()
|
#51 0x0000000a5d643734 in ?? ()
|
#52 0x00007fe15c0f5f43 in memcpy (__len=<optimized out>, __src=<optimized out>, __dest=0x7fe112144d56)
|
at /usr/include/x86_64-linux-gnu/bits/string3.h:51
|
#53 StoredValueFactory::newStoredValue (this=<optimized out>, itm=..., n=0x7fe15c1a12df, ht=..., setDirty=<optimized out>)
|
at /home/couchbase/serverjenkins/workspace/watson-unix/ep-engine/src/stored-value.h:778
|
#54 0x00007fe15c0f62c5 in HashTable::unlocked_set (this=0x7fe09aae9660, v=@0x100000000: <error reading variable>, val=...,
|
cas=<optimized out>, allowExisting=<optimized out>, hasMetaData=<optimized out>, policy=FULL_EVICTION, nru=0 '\000',
|
maybeKeyExists=false, isReplication=true)
|
at /home/couchbase/serverjenkins/workspace/watson-unix/ep-engine/src/stored-value.h:1123
|
#55 0x00007fe15c0eead5 in EventuallyPersistentStore::setWithMeta (this=<optimized out>, itm=..., cas=cas@entry=0,
|
seqno=seqno@entry=0x0, cookie=<optimized out>, force=force@entry=true, allowExisting=allowExisting@entry=true,
|
nru=nru@entry=2 '\002', genBySeqno=genBySeqno@entry=false, emd=0x7fe09b6bcee0, isReplication=isReplication@entry=true)
|
at /home/couchbase/serverjenkins/workspace/watson-unix/ep-engine/src/ep.cc:2356
|
#56 0x00007fe15c0dbf24 in PassiveStream::processMutation (this=this@entry=0x7fe123646280, mutation=mutation@entry=0x7fe09b2ea820)
|
at /home/couchbase/serverjenkins/workspace/watson-unix/ep-engine/src/dcp/stream.cc:1490
|
#57 0x00007fe15c0dc87b in PassiveStream::messageReceived (this=0x7fe123646280, resp=resp@entry=0x7fe09b2ea820)
|
at /home/couchbase/serverjenkins/workspace/watson-unix/ep-engine/src/dcp/stream.cc:1351
|
#58 0x00007fe15c0c6d7f in DcpConsumer::mutation (this=0x7fe12397e200, opaque=<optimized out>, key=<optimized out>, nkey=20,
|
value=<optimized out>, nvalue=2585, cas=1454084903182532608, vbucket=1009, flags=2, datatype=1 '\001', locktime=0,
|
bySeqno=102955, revSeqno=28309, exptime=0, nru=2 '\002', meta=0x7fe1237bbc2a, nmeta=5)
|
at /home/couchbase/serverjenkins/workspace/watson-unix/ep-engine/src/dcp/consumer.cc:291
|
#59 0x00007fe15c0fc8ea in EvpDcpMutation (handle=<optimized out>, cookie=0x7fe12390d000, opaque=208, key=0x7fe1237bb1fd,
|
nkey=<optimized out>, value=0x7fe1237bb211, nvalue=2585, cas=1454084903182532608, vbucket=1009, flags=2, datatype=1 '\001',
|
bySeqno=102955, revSeqno=28309, expiration=0, lockTime=0, meta=0x7fe1237bbc2a, nmeta=5, nru=2 '\002')
|
at /home/couchbase/serverjenkins/workspace/watson-unix/ep-engine/src/ep_engine.cc:1612
|
#60 0x0000000000428d3a in dcp_mutation_executor (c=c@entry=0x7fe12390d000, packet=packet@entry=0x7fe1237bb1c6)
|
at /home/couchbase/serverjenkins/workspace/watson-unix/memcached/daemon/mcbp_executors.cc:2387
|
#61 0x000000000043078a in process_bin_packet (c=0x7fe12390d000)
|
at /home/couchbase/serverjenkins/workspace/watson-unix/memcached/daemon/mcbp_executors.cc:4705
|
#62 mcbp_complete_nread (c=c@entry=0x7fe12390d000)
|
at /home/couchbase/serverjenkins/workspace/watson-unix/memcached/daemon/mcbp_executors.cc:4804
|
#63 0x0000000000446bac in conn_nread (c=0x7fe12390d000)
|
---Type <return> to continue, or q <return> to quit---
|
at /home/couchbase/serverjenkins/workspace/watson-unix/memcached/daemon/statemachine_mcbp.cc:305
|
#64 0x000000000041e308 in execute (connection=..., this=<optimized out>)
|
at /home/couchbase/serverjenkins/workspace/watson-unix/memcached/daemon/statemachine_mcbp.h:43
|
#65 McbpConnection::runStateMachinery (this=this@entry=0x7fe12390d000)
|
at /home/couchbase/serverjenkins/workspace/watson-unix/memcached/daemon/connection_mcbp.cc:1000
|
#66 0x000000000041e34b in McbpConnection::runEventLoop (this=0x7fe12390d000, which=<optimized out>)
|
at /home/couchbase/serverjenkins/workspace/watson-unix/memcached/daemon/connection_mcbp.cc:1266
|
#67 0x0000000000420970 in run_event_loop (c=0x7fe12390d000, which=<optimized out>)
|
at /home/couchbase/serverjenkins/workspace/watson-unix/memcached/daemon/connections.cc:143
|
#68 0x000000000043ab34 in event_handler (fd=<optimized out>, which=<optimized out>, arg=0x7fe12390d000)
|
at /home/couchbase/serverjenkins/workspace/watson-unix/memcached/daemon/memcached.cc:812
|
#69 0x00007fe160de66c8 in event_persist_closure (ev=<optimized out>, base=0x7fe15ea17300)
|
at /home/couchbase/jenkins/workspace/cbdeps-build/label/ubuntu14.04/release/sherlock/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1319
|
#70 event_process_active_single_queue (activeq=0x7fe15ead4330, base=0x7fe15ea17300)
|
at /home/couchbase/jenkins/workspace/cbdeps-build/label/ubuntu14.04/release/sherlock/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1363
|
#71 event_process_active (base=<optimized out>)
|
at /home/couchbase/jenkins/workspace/cbdeps-build/label/ubuntu14.04/release/sherlock/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1438
|
#72 event_base_loop (base=0x7fe15ea17300, flags=0)
|
at /home/couchbase/jenkins/workspace/cbdeps-build/label/ubuntu14.04/release/sherlock/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1639
|
#73 0x000000000044c3a4 in worker_libevent (arg=0x7fe158434fd8)
|
at /home/couchbase/serverjenkins/workspace/watson-unix/memcached/daemon/thread.cc:231
|
#74 0x00007fe16186553a in run (this=0x7fe15ea7dfa0)
|
at /home/couchbase/serverjenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
|
#75 platform_thread_wrap (arg=0x7fe15ea7dfa0)
|
at /home/couchbase/serverjenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
|
#76 0x00007fe16017b182 in start_thread (arg=0x7fe151bf2700) at pthread_create.c:312
|
#77 0x00007fe15f98e47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
(gdb)
|