Uploaded image for project: 'Couchbase Server'
  1. Couchbase Server
  2. MB-39671

[Jepsen] Memcached crash due to std::bad_alloc

    XMLWordPrintable

Details

    • Bug
    • Resolution: Incomplete
    • Major
    • 7.0.0
    • Cheshire-Cat
    • couchbase-bucket
    • Ubuntu 1804
      Couchbase version 7.0.0-2215
    • Triaged
    • 1
    • Unknown

    Description

      Memchaced crash due to std::bad_alloc found during kv-engine-cheshire-cat-jepsen-nightly-152 for test lein trampoline run test --nodes-file ./nodes --username root --password couchbase --workload=set-kill --scenario=hard-reboot --replicas=1 --no-autofailover --durability=0:0:100:0 --eviction-policy=value --enable-tcp-capture --hashdump --enable-memcached-debug-log-level --net-interface=eth0

      (gdb) bt
      #0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
      #1  0x00007fcb0ce54801 in __GI_abort () at abort.c:79
      #2  0x00007fcb0da72165 in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/vterminate.cc:95
      #3  0x000000000054b912 in backtrace_terminate_handler () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:86
      #4  0x00007fcb0da6ff56 in __cxxabiv1::__terminate (handler=<optimized out>) at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/eh_terminate.cc:47
      #5  0x00007fcb0da6ef19 in __cxa_call_terminate (ue_header=ue_header@entry=0x7fcaec002120) at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/eh_call.cc:54
      #6  0x00007fcb0da6f888 in __cxxabiv1::__gxx_personality_v0 (version=<optimized out>, actions=6, exception_class=5138137972254386944, ue_header=0x7fcaec002120, context=<optimized out>) at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/eh_personality.cc:676
      #7  0x00007fcb0d434343 in _Unwind_RaiseException_Phase2 (exc=exc@entry=0x7fcaec002120, context=context@entry=0x7fcb02ff25c0) at /tmp/deploy/objdir/../gcc-7.3.0/libgcc/unwind.inc:62
      #8  0x00007fcb0d434850 in _Unwind_RaiseException (exc=exc@entry=0x7fcaec002120) at /tmp/deploy/objdir/../gcc-7.3.0/libgcc/unwind.inc:131
      #9  0x00007fcb0da701d6 in __cxxabiv1::__cxa_throw (obj=<optimized out>, tinfo=0x7f8400 <typeinfo for std::bad_alloc>, dest=0x418cb0 <std::bad_alloc::~bad_alloc()@plt>) at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/eh_throw.cc:88
      #10 0x00007fcb1173eef1 in operator new (count=count@entry=18446744073709551600) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/global_new_replacement.cc:73
      #11 0x00007fcb113adacf in __gnu_cxx::new_allocator<cb::tracing::Span>::allocate (this=<optimized out>, __n=<optimized out>) at /usr/local/include/c++/7.3.0/ext/new_allocator.h:111
      #12 std::allocator_traits<std::allocator<cb::tracing::Span> >::allocate (__a=..., __n=<optimized out>) at /usr/local/include/c++/7.3.0/bits/alloc_traits.h:436
      #13 std::_Vector_base<cb::tracing::Span, std::allocator<cb::tracing::Span> >::_M_allocate (this=<optimized out>, __n=<optimized out>) at /usr/local/include/c++/7.3.0/bits/stl_vector.h:172
      #14 std::vector<cb::tracing::Span, std::allocator<cb::tracing::Span> >::_M_realloc_insert<cb::tracing::Code&, std::chrono::time_point<std::chrono::_V2::steady_clock, std::chrono::duration<long, std::ratio<1l, 1000000000l> > >&> (__position=..., this=0x7fcacbd0a810) at /usr/local/include/c++/7.3.0/bits/vector.tcc:406
      #15 std::vector<cb::tracing::Span, std::allocator<cb::tracing::Span> >::emplace_back<cb::tracing::Code&, std::chrono::time_point<std::chrono::_V2::steady_clock, std::chrono::duration<long, std::ratio<1l, 1000000000l> > >&> (this=0x7fcacbd0a810, this@entry=0x7fcb0c9a8780) at /usr/local/include/c++/7.3.0/bits/vector.tcc:105
      #16 cb::tracing::Tracer::begin (this=this@entry=0x7fcacbd0a810, tracecode=<optimized out>, startTime=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/tracing/tracer.cc:35
      #17 0x00007fcb11215e7f in TracerStopwatch::~TracerStopwatch (this=0x7fcb02ff2940, __in_chrg=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/trace_helpers.h:87
      #18 0x00007fcb1120f40f in ActiveDurabilityMonitor::State::advanceAndAckForPosition (this=this@entry=0x7fcafc1abf80, pos=..., node=..., shouldAck=shouldAck@entry=true) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/durability/active_durability_monitor.cc:654
      #19 0x00007fcb1120fdbb in ActiveDurabilityMonitor::State::advanceNodePosition (this=this@entry=0x7fcafc1abf80, node=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/durability/active_durability_monitor.cc:607
      #20 0x00007fcb11214c4c in ActiveDurabilityMonitor::State::processSeqnoAck (this=0x7fcafc1abf80, node=..., seqno=seqno@entry=484, toCommit=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/durability/active_durability_monitor.cc:944
      #21 0x00007fcb11214da7 in ActiveDurabilityMonitor::seqnoAckReceived (this=0x7fcafc1de150, replica=..., preparedSeqno=preparedSeqno@entry=484) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/durability/active_durability_monitor.cc:311
      #22 0x00007fcb112e0bc0 in VBucket::seqnoAcknowledged (this=<optimized out>, vbStateLock=..., replicaId=..., preparedSeqno=preparedSeqno@entry=484) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.cc:3930
      #23 0x00007fcb111b057c in ActiveStream::seqnoAck (this=this@entry=0x7fcafc3a6a90, consumerName=..., preparedSeqno=484) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/active_stream.cc:2109
      #24 0x00007fcb111fdaa5 in DcpProducer::seqno_acknowledged (this=0x7fcafc333910, opaque=2, vbucket=..., prepared_seqno=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/producer.cc:1144
      #25 0x00007fcb11245fd0 in non-virtual thunk to EventuallyPersistentEngine::seqno_acknowledged(gsl::not_null<void const*>, unsigned int, Vbid, unsigned long) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/gsl-lite/include/gsl/gsl-lite.h:473
      #26 0x00000000004541c5 in dcpSeqnoAcknowledged (cookie=..., opaque=2, vbucket=..., prepared_seqno=484) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:768
      #27 0x00000000004bcc3c in dcp_seqno_acknowledged_executor (cookie=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/dcp_seqno_acknowledged_executor.cc:33
      #28 0x00000000004aa026 in std::function<void (Cookie&)>::operator()(Cookie&) const (__args#0=..., this=<optimized out>) at /usr/local/include/c++/7.3.0/bits/std_function.h:706
      #29 execute_client_request_packet (cookie=..., request=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mcbp_executors.cc:897
      #30 0x00000000004990f9 in Cookie::execute (this=this@entry=0x7fcb09779c00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:153
      #31 0x0000000000481ccb in Connection::executeCommandPipeline (this=this@entry=0x7fcb096ab900) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:569
      #32 0x00000000004853c8 in Connection::executeCommandsCallback (this=this@entry=0x7fcb096ab900) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:638
      #33 0x0000000000485b05 in Connection::rw_callback (ctx=0x7fcb096ab900) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:762
      #34 0x00007fcb0ed4371e in bufferevent_run_deferred_callbacks_unlocked (cb=<optimized out>, arg=0x7fcaec0028a0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/bufferevent.c:208
      #35 0x00007fcb0ed4c691 in event_process_active_single_queue (base=base@entry=0x231aed0, max_to_process=max_to_process@entry=2147483647, endtime=endtime@entry=0x0, activeq=<optimized out>) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1726
      #36 0x00007fcb0ed4cedf in event_process_active (base=0x231aed0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1789
      #37 event_base_loop (base=0x231aed0, flags=flags@entry=0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:2012
      #38 0x000000000046a1c9 in worker_libevent (arg=0x7fcb0c9e7ff8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:210
      #39 0x00007fcb0fc7d387 in CouchbaseThread::run (this=0x7fcb096a7610) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58
      #40 platform_thread_wrap (arg=0x7fcb096a7610) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71
      #41 0x00007fcb0d20c6db in start_thread (arg=0x7fcb02ffd700) at pthread_create.c:463
      #42 0x00007fcb0cf3588f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
      

      (gdb) bt full
      #0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
              set = {__val = {0, 140510075291395, 140509519552720, 0, 140510075295920, 140509905362128, 386183668047020927, 229217556, 50274824, 281470681751456, 0, 0, 0, 0, 0, 0}}
              pid = <optimized out>
              tid = <optimized out>
              ret = <optimized out>
      #1  0x00007fcb0ce54801 in __GI_abort () at abort.c:79
              save_stage = 1
              act = {__sigaction_handler = {sa_handler = 0x3d7168e143202a0b, sa_sigaction = 0x3d7168e143202a0b}, sa_mask = {__val = {0, 0, 0, 0, 0, 140510075291395, 140510071754577, 11, 140510075291264, 10, 140509519568944, 140509905363744, 0, 0, 140510071755779, 140510075291264}}, sa_flags = 229044560, sa_restorer = 0x7fcb0db15302}
              sigs = {__val = {32, 0 <repeats 15 times>}}
              __cnt = <optimized out>
              __set = <optimized out>
              __cnt = <optimized out>
              __set = <optimized out>
      #2  0x00007fcb0da72165 in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/vterminate.cc:95
              terminating = false
              t = <optimized out>
      #3  0x000000000054b912 in backtrace_terminate_handler () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:86
      No locals.
      #4  0x00007fcb0da6ff56 in __cxxabiv1::__terminate (handler=<optimized out>) at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/eh_terminate.cc:47
      No locals.
      #5  0x00007fcb0da6ef19 in __cxa_call_terminate (ue_header=ue_header@entry=0x7fcaec002120) at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/eh_call.cc:54
              xh = <optimized out>
      #6  0x00007fcb0da6f888 in __cxxabiv1::__gxx_personality_v0 (version=<optimized out>, actions=6, exception_class=5138137972254386944, ue_header=0x7fcaec002120, context=<optimized out>) at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/eh_personality.cc:676
              found_type = <optimized out>
              info = {Start = 0, LPStart = 140509905363672, ttype_base = 666009389144, TType = 0x7fcb1140c92d "\f\a\b\220\001", action_table = 0x7fcb02ff2320 "", ttype_encoding = 172 '\254', call_site_encoding = 95 '_'}
              language_specific_data = <optimized out>
              action_record = <optimized out>
              p = <optimized out>
              landing_pad = <optimized out>
              ip = <optimized out>
              handler_switch_value = <optimized out>
              thrown_ptr = 0x0
              foreign_exception = <optimized out>
              ip_before_insn = 0
              xh = <optimized out>
      #7  0x00007fcb0d434343 in _Unwind_RaiseException_Phase2 (exc=exc@entry=0x7fcaec002120, context=context@entry=0x7fcb02ff25c0) at /tmp/deploy/objdir/../gcc-7.3.0/libgcc/unwind.inc:62
              fs = {regs = {reg = {{loc = {reg = 0, offset = 0, exp = 0x0}, how = REG_UNSAVED}, {loc = {reg = 0, offset = 0, exp = 0x0}, how = REG_UNSAVED}, {loc = {reg = 0, offset = 0, exp = 0x0}, how = REG_UNSAVED}, {loc = {reg = 18446744073709551584, offset = -32, exp = 0xffffffffffffffe0 <error: Cannot access memory at address 0xffffffffffffffe0>}, 
                      how = REG_SAVED_OFFSET}, {loc = {reg = 0, offset = 0, exp = 0x0}, how = REG_UNSAVED}, {loc = {reg = 0, offset = 0, exp = 0x0}, how = REG_UNSAVED}, {loc = {reg = 18446744073709551592, offset = -24, exp = 0xffffffffffffffe8 <error: Cannot access memory at address 0xffffffffffffffe8>}, how = REG_SAVED_OFFSET}, {loc = {reg = 0, offset = 0, 
                        exp = 0x0}, how = REG_UNSAVED}, {loc = {reg = 0, offset = 0, exp = 0x0}, how = REG_UNSAVED}, {loc = {reg = 0, offset = 0, exp = 0x0}, how = REG_UNSAVED}, {loc = {reg = 0, offset = 0, exp = 0x0}, how = REG_UNSAVED}, {loc = {reg = 0, offset = 0, exp = 0x0}, how = REG_UNSAVED}, {loc = {reg = 18446744073709551600, offset = -16, 
                        exp = 0xfffffffffffffff0 <error: Cannot access memory at address 0xfffffffffffffff0>}, how = REG_SAVED_OFFSET}, {loc = {reg = 0, offset = 0, exp = 0x0}, how = REG_UNSAVED}, {loc = {reg = 0, offset = 0, exp = 0x0}, how = REG_UNSAVED}, {loc = {reg = 0, offset = 0, exp = 0x0}, how = REG_UNSAVED}, {loc = {reg = 18446744073709551608, 
                        offset = -8, exp = 0xfffffffffffffff8 <error: Cannot access memory at address 0xfffffffffffffff8>}, how = REG_SAVED_OFFSET}, {loc = {reg = 0, offset = 0, exp = 0x0}, how = REG_UNSAVED}}, prev = 0x0, cfa_offset = 48, cfa_reg = 7, cfa_exp = 0x0, cfa_how = CFA_REG_OFFSET}, pc = 0x7fcb11215e9a <TracerStopwatch::~TracerStopwatch()+90>, 
                personality = 0x419bc0 <__gxx_personality_v0@plt>, data_align = -8, code_align = 1, retaddr_column = 16, fde_encoding = 27 '\033', lsda_encoding = 27 '\033', saw_z = 1 '\001', signal_frame = 0 '\000', eh_ptr = 0x0}
              match_handler = 4
              code = _URC_NO_REASON
      #8  0x00007fcb0d434850 in _Unwind_RaiseException (exc=exc@entry=0x7fcaec002120) at /tmp/deploy/objdir/../gcc-7.3.0/libgcc/unwind.inc:131
              this_context = {reg = {0x7fcb02ff2838, 0x7fcb02ff2840, 0x0, 0x7fcb02ff2848, 0x0, 0x0, 0x7fcb02ff2870, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7fcb02ff2850, 0x7fcb02ff2858, 0x7fcb02ff2860, 0x7fcb02ff2868, 0x7fcb02ff2878, 0x0}, cfa = 0x7fcb02ff2880, ra = 0x7fcb0da701d6 <__cxxabiv1::__cxa_throw(void*, std::type_info*, void (*)(void*))+54>, lsda = 0x0, bases = {
                  tbase = 0x0, dbase = 0x0, func = 0x7fcb0d4345a0 <_Unwind_RaiseException>}, flags = 4611686018427387904, version = 0, args_size = 0, by_value = '\000' <repeats 17 times>}
              cur_context = {reg = {0x7fcb02ff2838, 0x7fcb02ff2840, 0x0, 0x7fcb02ff28c8, 0x0, 0x0, 0x7fcb02ff28d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7fcb02ff28d8, 0x7fcb02ff28e0, 0x7fcb02ff28e8, 0x7fcb02ff28f0, 0x7fcb02ff28f8, 0x0}, cfa = 0x7fcb02ff2900, ra = 0x7fcb11215e7f <TracerStopwatch::~TracerStopwatch()+63>, lsda = 0x7fcb11465fac, bases = {tbase = 0x0, 
                  dbase = 0x0, func = 0x7fcb11215e40 <TracerStopwatch::~TracerStopwatch()>}, flags = 4611686018427387904, version = 0, args_size = 0, by_value = '\000' <repeats 17 times>}
              code = <optimized out>
      #9  0x00007fcb0da701d6 in __cxxabiv1::__cxa_throw (obj=<optimized out>, tinfo=0x7f8400 <typeinfo for std::bad_alloc>, dest=0x418cb0 <std::bad_alloc::~bad_alloc()@plt>) at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/eh_throw.cc:88
              globals = <optimized out>
              header = 0x7fcaec0020c0
      #10 0x00007fcb1173eef1 in operator new (count=count@entry=18446744073709551600) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/global_new_replacement.cc:73
              result = <optimized out>
      #11 0x00007fcb113adacf in __gnu_cxx::new_allocator<cb::tracing::Span>::allocate (this=<optimized out>, __n=<optimized out>) at /usr/local/include/c++/7.3.0/ext/new_allocator.h:111
              __al = <optimized out>
      #12 std::allocator_traits<std::allocator<cb::tracing::Span> >::allocate (__a=..., __n=<optimized out>) at /usr/local/include/c++/7.3.0/bits/alloc_traits.h:436
      No locals.
      #13 std::_Vector_base<cb::tracing::Span, std::allocator<cb::tracing::Span> >::_M_allocate (this=<optimized out>, __n=<optimized out>) at /usr/local/include/c++/7.3.0/bits/stl_vector.h:172
      No locals.
      #14 std::vector<cb::tracing::Span, std::allocator<cb::tracing::Span> >::_M_realloc_insert<cb::tracing::Code&, std::chrono::time_point<std::chrono::_V2::steady_clock, std::chrono::duration<long, std::ratio<1l, 1000000000l> > >&> (__position=..., this=0x7fcacbd0a810) at /usr/local/include/c++/7.3.0/bits/vector.tcc:406
              __len = <optimized out>
              __elems_before = <optimized out>
              __new_start = <optimized out>
              __new_finish = <optimized out>
              __len = <optimized out>
              __elems_before = <optimized out>
              __new_start = <optimized out>
              __new_finish = <optimized out>
      #15 std::vector<cb::tracing::Span, std::allocator<cb::tracing::Span> >::emplace_back<cb::tracing::Code&, std::chrono::time_point<std::chrono::_V2::steady_clock, std::chrono::duration<long, std::ratio<1l, 1000000000l> > >&> (this=0x7fcacbd0a810, this@entry=0x7fcb0c9a8780) at /usr/local/include/c++/7.3.0/bits/vector.tcc:105
      No locals.
      #16 cb::tracing::Tracer::begin (this=this@entry=0x7fcacbd0a810, tracecode=<optimized out>, startTime=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/tracing/tracer.cc:35
      No locals.
      #17 0x00007fcb11215e7f in TracerStopwatch::~TracerStopwatch (this=0x7fcb02ff2940, __in_chrg=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/trace_helpers.h:87
              guard = {engine = 0x7fcb08000000}
              tracer = <optimized out>
              spanId = <optimized out>
              this = 0x7fcb02ff2940
              __in_chrg = <optimized out>
              guard = <optimized out>
              tracer = <optimized out>
              spanId = <optimized out>
      #18 0x00007fcb1120f40f in ActiveDurabilityMonitor::State::advanceAndAckForPosition (this=this@entry=0x7fcafc1abf80, pos=..., node=..., shouldAck=shouldAck@entry=true) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/durability/active_durability_monitor.cc:654
              ackTime = {__d = {__r = 82579320322363}}
              event = <optimized out>
              ackTimer = {traceable = 0x7fcacbd0a800, code = cb::tracing::Code::SyncWriteAckRemote, startTime = {__d = {__r = 82579320322363}}, stopTime = {__d = {__r = 82579320322363}}}
              cookie = 0x7fcacbd0a800
      #19 0x00007fcb1120fdbb in ActiveDurabilityMonitor::State::advanceNodePosition (this=this@entry=0x7fcafc1abf80, node=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/durability/active_durability_monitor.cc:607
              pos = <error reading variable>
              firstChainItr = {<std::__detail::_Node_iterator_base<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, DurabilityMonitor::Position<std::__cxx11::list<DurabilityMonitor::ActiveSyncWrite, std::allocator<DurabilityMonitor::ActiveSyncWrite> > > >, true>> = {_M_cur = 0x7fcafc21bc00}, <No data fields>}
              firstChainFound = <optimized out>
              __func__ = "advanceNodePosition"
              secondChainItr = {<std::__detail::_Node_iterator_base<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, DurabilityMonitor::Position<std::__cxx11::list<DurabilityMonitor::ActiveSyncWrite, std::allocator<DurabilityMonitor::ActiveSyncWrite> > > >, true>> = {_M_cur = 0x0}, <No data fields>}
              secondChainFound = false
      #20 0x00007fcb11214c4c in ActiveDurabilityMonitor::State::processSeqnoAck (this=0x7fcafc1abf80, node=..., seqno=seqno@entry=484, toCommit=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/durability/active_durability_monitor.cc:944
              posIt = <optimized out>
              next = <optimized out>
      #21 0x00007fcb11214da7 in ActiveDurabilityMonitor::seqnoAckReceived (this=0x7fcafc1de150, replica=..., preparedSeqno=preparedSeqno@entry=484) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/durability/active_durability_monitor.cc:311
      ---Type <return> to continue, or q <return> to quit---
      No locals.
      #22 0x00007fcb112e0bc0 in VBucket::seqnoAcknowledged (this=<optimized out>, vbStateLock=..., replicaId=..., preparedSeqno=preparedSeqno@entry=484) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.cc:3930
      No locals.
      #23 0x00007fcb111b057c in ActiveStream::seqnoAck (this=this@entry=0x7fcafc3a6a90, consumerName=..., preparedSeqno=484) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/active_stream.cc:2109
              lh = {_M_device = @0x7fcafc3a6af8}
              vbStateLh = {lock_ = 0x7fcafc1a6dd8, token_ = {type_ = folly::SharedMutexToken::Type::DEFERRED_SHARED, slot_ = 1}}
              vb = {<std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<VBucket, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7fcafc1a6c00, _M_refcount = {_M_pi = 0x7fcafc1eb220}}, <No data fields>}
      #24 0x00007fcb111fdaa5 in DcpProducer::seqno_acknowledged (this=0x7fcafc333910, opaque=2, vbucket=..., prepared_seqno=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/producer.cc:1144
              vb = {<std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<VBucket, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7fcafc1a6c00, _M_refcount = {_M_pi = 0x7fcafc1eb220}}, <No data fields>}
              rv = {<folly::detail::IteratorFacade<folly::AtomicHashMap<unsigned short, std::shared_ptr<StreamContainer<std::shared_ptr<Stream> > > >::ahm_iterator<folly::AtomicHashMap<unsigned short, std::shared_ptr<StreamContainer<std::shared_ptr<Stream> > >, std::hash<unsigned short>, std::equal_to<unsigned short>, std::allocator<char>, folly::AtomicHashArrayLinearProbeFcn, folly::Identity>, std::pair<unsigned short const, std::shared_ptr<StreamContainer<std::shared_ptr<Stream> > > >, folly::AtomicHashArray<unsigned short, std::shared_ptr<StreamContainer<std::shared_ptr<Stream> > >, std::hash<unsigned short>, std::equal_to<unsigned short>, std::allocator<char>, folly::AtomicHashArrayLinearProbeFcn, folly::Identity>::aha_iterator<folly::AtomicHashArray<unsigned short, std::shared_ptr<StreamContainer<std::shared_ptr<Stream> > >, std::hash<unsigned short>, std::equal_to<unsigned short>, std::allocator<char>, folly::AtomicHashArrayLinearProbeFcn, folly::Identity>, std::pair<unsigned short const, std::shared_ptr<StreamContainer<std::shared_ptr<Stream> > > > > >, std::pair<unsigned short const, std::shared_ptr<StreamContainer<std::shared_ptr<Stream> > > >, std::forward_iterator_tag>> = {<No data fields>}, ahm_ = 0x7fcafc333b40, subMap_ = 0, 
                subIt_ = {<folly::detail::IteratorFacade<folly::AtomicHashArray<unsigned short, std::shared_ptr<StreamContainer<std::shared_ptr<Stream> > >, std::hash<unsigned short>, std::equal_to<unsigned short>, std::allocator<char>, folly::AtomicHashArrayLinearProbeFcn, folly::Identity>::aha_iterator<folly::AtomicHashArray<unsigned short, std::shared_ptr<StreamContainer<std::shared_ptr<Stream> > >, std::hash<unsigned short>, std::equal_to<unsigned short>, std::allocator<char>, folly::AtomicHashArrayLinearProbeFcn, folly::Identity>, std::pair<unsigned short const, std::shared_ptr<StreamContainer<std::shared_ptr<Stream> > > > >, std::pair<unsigned short const, std::shared_ptr<StreamContainer<std::shared_ptr<Stream> > > >, std::forward_iterator_tag>> = {<No data fields>}, aha_ = 0x7fcafc2b7000, offset_ = 54}}
              stream = {<std::__shared_ptr<ActiveStream, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<ActiveStream, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7fcafc3a6a90, _M_refcount = {_M_pi = 0x7fcafc3a6a80}}, <No data fields>}
      #25 0x00007fcb11245fd0 in non-virtual thunk to EventuallyPersistentEngine::seqno_acknowledged(gsl::not_null<void const*>, unsigned int, Vbid, unsigned long) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/gsl-lite/include/gsl/gsl-lite.h:473
              cb::xattr::vattrs::VBUCKET = <optimized out>
              cb::userdataStartTag = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x0}, _M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}}
              cb::rbac::PrivilegeAccessFail = <optimized out>
              Collections::DefaultScopeName = <optimized out>
              folly::sorted_equivalent = <optimized out>
              nlohmann::(anonymous namespace)::from_json = <optimized out>
              cb::rbac::PrivilegeAccessOk = <optimized out>
              spdlog::level::short_level_names = <optimized out>
              Collections::DefaultCollectionName = <optimized out>
              NoopSeqnoAckCb = {<std::_Maybe_unary_or_binary_function<void, Vbid, long>> = {<std::binary_function<Vbid, long, void>> = {<No data fields>}, <No data fields>}, <std::_Function_base> = {static _M_max_size = 16, static _M_max_align = 8, _M_functor = {_M_unused = {_M_object = 0x0, _M_const_object = 0x0, _M_function_pointer = 0x0, 
                      _M_member_pointer = NULL}, _M_pod_data = '\000' <repeats 15 times>}, _M_manager = 0x0}, _M_invoker = 0x0}
              Collections::CollectionEventDebugTag = <optimized out>
              nlohmann::(anonymous namespace)::to_json = <optimized out>
              cb::xattr::macros::VALUE_CRC32C = <optimized out>
              NoopSyncWriteCompleteCb = {<std::_Maybe_unary_or_binary_function<void, void const*, ENGINE_ERROR_CODE>> = {<std::binary_function<void const*, ENGINE_ERROR_CODE, void>> = {<No data fields>}, <No data fields>}, <std::_Function_base> = {static _M_max_size = 16, static _M_max_align = 8, _M_functor = {_M_unused = {_M_object = 0x0, _M_const_object = 0x0, 
                      _M_function_pointer = 0x0, _M_member_pointer = NULL}, _M_pod_data = '\000' <repeats 15 times>}, _M_manager = 0x0}, _M_invoker = 0x0}
              std::__ioinit = {static _S_refcount = 0, static _S_synced_with_stdio = true}
              cb::xattr::macros::SEQNO = <optimized out>
              aliveVBStates = {bits = {<std::_Base_bitset<1>> = {_M_w = 0}, <No data fields>}}
              folly::sorted_unique = <optimized out>
              spdlog::details::days = <optimized out>
              globalBucketLoggerName = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x0}, _M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}}
              makeExitBorderGuard = <optimized out>
              Collections::ScopeEventDebugTag = <optimized out>
              spdlog::details::os::default_eol = <optimized out>
              spdlog::level::level_names = <optimized out>
              spdlog::details::months = <optimized out>
              cb::rbac::PrivilegeAccessFailNoPrivileges = <optimized out>
              cb::userdataEndTag = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x0}, _M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}}
              Collections::DefaultScopeIdentifier = <optimized out>
              spdlog::details::full_days = <optimized out>
              spdlog::details::full_months = <optimized out>
              cb::xattr::vattrs::DOCUMENT = <optimized out>
              cb::xattr::vattrs::XTOC = <optimized out>
              Collections::DefaultCollectionIdentifier = <optimized out>
              cb::xattr::macros::CAS = <optimized out>
              flatbuffers::flatbuffer_version_string = 0x2935a3 <error: Cannot access memory at address 0x2935a3>
              fmt::v5::internal::basic_data<void>::DIGITS = 0x7fcb11692f60 <fmt::v5::internal::basic_data<void>::DIGITS> "00010203040506070809101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899"
              fmt::v5::internal::basic_data<void>::POWERS_OF_10_32 = 0x7fcb116930e0 <fmt::v5::internal::basic_data<void>::POWERS_OF_10_32>
              fmt::v5::internal::basic_data<void>::POWERS_OF_10_64 = 0x7fcb11693040 <fmt::v5::internal::basic_data<void>::POWERS_OF_10_64>
      #26 0x00000000004541c5 in dcpSeqnoAcknowledged (cookie=..., opaque=2, vbucket=..., prepared_seqno=484) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:768
              connection = <optimized out>
              dcp = <optimized out>
              ret = <optimized out>
      #27 0x00000000004bcc3c in dcp_seqno_acknowledged_executor (cookie=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/dcp_seqno_acknowledged_executor.cc:33
              req = <error reading variable>
              extdata = <optimized out>
              extras = <optimized out>
              ret = <optimized out>
              connection = <optimized out>
      #28 0x00000000004aa026 in std::function<void (Cookie&)>::operator()(Cookie&) const (__args#0=..., this=<optimized out>) at /usr/local/include/c++/7.3.0/bits/std_function.h:706
      No locals.
      #29 execute_client_request_packet (cookie=..., request=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mcbp_executors.cc:897
              c = <optimized out>
              privilegeChains = {commandChains = {_M_elems = {{chain = {<std::_Vector_base<Function<cb::rbac::PrivilegeAccess, cb::rbac::PrivilegeAccess::getSuccessValue, Cookie&>, std::allocator<Function<cb::rbac::PrivilegeAccess, cb::rbac::PrivilegeAccess::getSuccessValue, Cookie&> > >> = {
                          _M_impl = {<std::allocator<Function<cb::rbac::PrivilegeAccess, cb::rbac::PrivilegeAccess::getSuccessValue, Cookie&> >> = {<__gnu_cxx::new_allocator<Function<cb::rbac::PrivilegeAccess, cb::rbac::PrivilegeAccess::getSuccessValue, Cookie&> >> = {<No data fields>}, <No data fields>}, _M_start = 0x0, _M_finish = 0x0, 
                            _M_end_of_storage = 0x0}}, <No data fields>}} <repeats 256 times>}}}
              opcode = <optimized out>
              res = {status = cb::rbac::PrivilegeAccess::Status::Ok}
      #30 0x00000000004990f9 in Cookie::execute (this=this@entry=0x7fcb09779c00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:153
              header = <error reading variable>
      #31 0x0000000000481ccb in Connection::executeCommandPipeline (this=this@entry=0x7fcb096ab900) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:569
              cookie = {_M_t = {_M_t = {<std::_Tuple_impl<0, Cookie*, std::default_delete<Cookie> >> = {<std::_Tuple_impl<1, std::default_delete<Cookie> >> = {<std::_Head_base<1, std::default_delete<Cookie>, true>> = {<std::default_delete<Cookie>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, Cookie*, false>> = {
                        _M_head_impl = 0x7fcb09779c00}, <No data fields>}, <No data fields>}}}
              drainSize = <optimized out>
              status = <optimized out>
              input = 0x7fcaec002ac0
              stop = false
              maxActiveCommands = 32
              active = false
              maxSendQueueSize = <optimized out>
      #32 0x00000000004853c8 in Connection::executeCommandsCallback (this=this@entry=0x7fcb096ab900) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:638
              start = <optimized out>
              stop = <optimized out>
              ns = <optimized out>
      #33 0x0000000000485b05 in Connection::rw_callback (ctx=0x7fcb096ab900) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:762
              instance = <error reading variable>
              thread = <optimized out>
              phosphor_internal_category_enabled_741 = {_M_b = {
      Segmentation fault (core dumped)
      

      Attachments

        No reviews matched the request. Check your Options in the drop-down menu of this sections header.

        Activity

          People

            richard.demellow Richard deMellow
            richard.demellow Richard deMellow
            Votes:
            0 Vote for this issue
            Watchers:
            3 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved:

              Gerrit Reviews

                There are no open Gerrit changes

                PagerDuty