Loading...

Details

Type: Bug
Resolution: Duplicate
Priority: Critical
Fix Version/s: 7.0.0
Affects Version/s: Cheshire-Cat
Component/s: couchbase-bucket
Labels:
None
Environment:
Ubuntu 1804
Couchbase Server 7.0.0-2047

Triage:
Untriaged
Story Points:
1
Is this a Regression?:
Unknown

Description

Linearizability failure and crash during kv-engine-cheshire-cat-jepsen-nightly-139 for test lein trampoline run test --nodes-file ./nodes --username root --password couchbase --workload=set-kill --scenario=suspend-process --process-to-suspend=memcached --process-suspend-time=10 --replicas=2 --autofailover --durability=0:0:100:0 --eviction-policy=value --enable-tcp-capture --hashdump --enable-memcached-debug-log-level --net-interface=eth0.

(gdb) bt

#0  0x00007ffa1f73f2c8 in std::_Construct<cb::tracing::Span, cb::tracing::Span> (__p=<optimized out>) at /usr/local/include/c++/7.3.0/bits/stl_uninitialized.h:82

#1  std::__uninitialized_copy<false>::__uninit_copy<std::move_iterator<cb::tracing::Span*>, cb::tracing::Span*> (__result=<optimized out>, __last=..., __first=...) at /usr/local/include/c++/7.3.0/bits/stl_uninitialized.h:83

#2  std::uninitialized_copy<std::move_iterator<cb::tracing::Span*>, cb::tracing::Span*> (__result=<optimized out>, __last=..., __first=...) at /usr/local/include/c++/7.3.0/bits/stl_uninitialized.h:134

#3  std::__uninitialized_copy_a<std::move_iterator<cb::tracing::Span*>, cb::tracing::Span*, cb::tracing::Span> (__result=<optimized out>, __last=..., __first=...) at /usr/local/include/c++/7.3.0/bits/stl_uninitialized.h:289

#4  std::__uninitialized_move_if_noexcept_a<cb::tracing::Span*, cb::tracing::Span*, std::allocator<cb::tracing::Span> > (__alloc=..., __result=<optimized out>, __last=0x7ffa178ae700, __first=0x7ffa17b73d00) at /usr/local/include/c++/7.3.0/bits/stl_uninitialized.h:312

#5  std::vector<cb::tracing::Span, std::allocator<cb::tracing::Span> >::_M_realloc_insert<cb::tracing::Code&, std::chrono::time_point<std::chrono::_V2::steady_clock, std::chrono::duration<long, std::ratio<1l, 1000000000l> > >&> (__position=..., this=0x7ffa178c8210) at /usr/local/include/c++/7.3.0/bits/vector.tcc:424

#6  std::vector<cb::tracing::Span, std::allocator<cb::tracing::Span> >::emplace_back<cb::tracing::Code&, std::chrono::time_point<std::chrono::_V2::steady_clock, std::chrono::duration<long, std::ratio<1l, 1000000000l> > >&> (this=0x7ffa178c8210, this@entry=0x7ffa178ae700) at /usr/local/include/c++/7.3.0/bits/vector.tcc:105

#7  cb::tracing::Tracer::begin (this=this@entry=0x7ffa178c8210, tracecode=<optimized out>, startTime=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/tracing/tracer.cc:35

#8  0x00007ffa1f5a7bef in TracerStopwatch::~TracerStopwatch (this=0x7ffa1686dc28, __in_chrg=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/trace_helpers.h:87

#9  0x00007ffa1f5cb756 in ScopeTimer2<MicrosecondStopwatch<Hdr1sfMicroSecHistogram>, TracerStopwatch>::~ScopeTimer2 (this=0x7ffa1686dc10, __in_chrg=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/include/platform/scope_timer.h:99

#10 EventuallyPersistentEngine::storeIfInner(void const*, Item&, unsigned long, ENGINE_STORE_OPERATION, std::function<cb::StoreIfStatus (std::optional<item_info> const&, cb::vbucket_info)> const&, bool) (this=this@entry=0x7ffa103ff000, cookie=cookie@entry=0x7ffa178c8200, item=..., cas=cas@entry=0, operation=operation@entry=OPERATION_ADD, predicate=...,

    preserveTtl=false) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:2557

#11 0x00007ffa1f5cbb28 in EventuallyPersistentEngine::store_if(gsl::not_null<void const*>, gsl::not_null<void*>, unsigned long, ENGINE_STORE_OPERATION, std::function<cb::StoreIfStatus (std::optional<item_info> const&, cb::vbucket_info)> const&, std::optional<cb::durability::Requirements> const&, DocumentState, bool) (this=0x7ffa103ff000, cookie=..., itm=...,

    cas=0, operation=OPERATION_ADD, predicate=..., durability=..., document_state=(DocumentState::Alive | unknown: -256), preserveTtl=false) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:448

#12 0x0000000000452b5c in bucket_store_if(Cookie&, gsl::not_null<void*>, unsigned long, ENGINE_STORE_OPERATION, std::function<cb::StoreIfStatus (std::optional<item_info> const&, cb::vbucket_info)>, std::optional<cb::durability::Requirements>, DocumentState, bool) (cookie=..., item_=..., cas=0, operation=OPERATION_ADD, predicate=..., durability=...,

    document_state=(DocumentState::Alive | unknown: -256), preserveTtl=false) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:145

#13 0x00000000004c4ce7 in MutationCommandContext::storeItem (this=this@entry=0x7ffa17a1f300) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/mutation_context.cc:294

#14 0x00000000004c5ab8 in MutationCommandContext::step (this=0x7ffa17a1f300) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/mutation_context.cc:60

#15 0x00000000004cf250 in SteppableCommandContext::drive (this=this@entry=0x7ffa17a1f300) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/steppable_command_context.cc:37

#16 0x00000000004a890d in add_set_replace_executor (store_op=OPERATION_ADD, cookie=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mcbp_executors.cc:163

#17 add_executor (cookie=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mcbp_executors.cc:167

#18 0x00000000004a9eb6 in std::function<void (Cookie&)>::operator()(Cookie&) const (__args#0=..., this=<optimized out>) at /usr/local/include/c++/7.3.0/bits/std_function.h:706

#19 execute_client_request_packet (cookie=..., request=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mcbp_executors.cc:897

#20 0x0000000000498f89 in Cookie::execute (this=this@entry=0x7ffa178c8200) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:153

#21 0x0000000000481b5b in Connection::executeCommandPipeline (this=this@entry=0x7ffa1ab87100) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:569

#22 0x0000000000485258 in Connection::executeCommandsCallback (this=this@entry=0x7ffa1ab87100) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:638

#23 0x0000000000485995 in Connection::rw_callback (ctx=0x7ffa1ab87100) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:762

#24 0x00007ffa1d0e871e in bufferevent_run_deferred_callbacks_unlocked (cb=<optimized out>, arg=0x7ffa0c003fc0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/bufferevent.c:208

#25 0x00007ffa1d0f1691 in event_process_active_single_queue (base=base@entry=0x15bc380, max_to_process=max_to_process@entry=2147483647, endtime=endtime@entry=0x0, activeq=<optimized out>) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1726

#26 0x00007ffa1d0f1edf in event_process_active (base=0x15bc380) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1789

#27 event_base_loop (base=0x15bc380, flags=flags@entry=0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:2012

#28 0x000000000046a0a9 in worker_libevent (arg=0x7ffa1abf9aa8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:210

#29 0x00007ffa1e022437 in CouchbaseThread::run (this=0x7ffa1aa3cd70) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58

#30 platform_thread_wrap (arg=0x7ffa1aa3cd70) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71

#31 0x00007ffa1b5b16db in start_thread (arg=0x7ffa16878700) at pthread_create.c:463

#32 0x00007ffa1b2da88f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

(gdb) bt full

#0  0x00007ffa1f73f2c8 in std::_Construct<cb::tracing::Span, cb::tracing::Span> (__p=<optimized out>) at /usr/local/include/c++/7.3.0/bits/stl_uninitialized.h:82

No locals.

#1  std::__uninitialized_copy<false>::__uninit_copy<std::move_iterator<cb::tracing::Span*>, cb::tracing::Span*> (__result=<optimized out>, __last=..., __first=...) at /usr/local/include/c++/7.3.0/bits/stl_uninitialized.h:83

        __cur = 0x7ffa17a5f800

        __cur = <optimized out>

#2  std::uninitialized_copy<std::move_iterator<cb::tracing::Span*>, cb::tracing::Span*> (__result=<optimized out>, __last=..., __first=...) at /usr/local/include/c++/7.3.0/bits/stl_uninitialized.h:134

        __assignable = <optimized out>

        __assignable = <optimized out>

#3  std::__uninitialized_copy_a<std::move_iterator<cb::tracing::Span*>, cb::tracing::Span*, cb::tracing::Span> (__result=<optimized out>, __last=..., __first=...) at /usr/local/include/c++/7.3.0/bits/stl_uninitialized.h:289

No locals.

#4  std::__uninitialized_move_if_noexcept_a<cb::tracing::Span*, cb::tracing::Span*, std::allocator<cb::tracing::Span> > (__alloc=..., __result=<optimized out>, __last=0x7ffa178ae700, __first=0x7ffa17b73d00) at /usr/local/include/c++/7.3.0/bits/stl_uninitialized.h:312

No locals.

#5  std::vector<cb::tracing::Span, std::allocator<cb::tracing::Span> >::_M_realloc_insert<cb::tracing::Code&, std::chrono::time_point<std::chrono::_V2::steady_clock, std::chrono::duration<long, std::ratio<1l, 1000000000l> > >&> (__position=..., this=0x7ffa178c8210) at /usr/local/include/c++/7.3.0/bits/vector.tcc:424

        __len = <optimized out>

        __elems_before = <optimized out>

        __new_start = <optimized out>

        __new_finish = 0x0

        __len = <optimized out>

        __elems_before = <optimized out>

        __new_start = <optimized out>

        __new_finish = <optimized out>

#6  std::vector<cb::tracing::Span, std::allocator<cb::tracing::Span> >::emplace_back<cb::tracing::Code&, std::chrono::time_point<std::chrono::_V2::steady_clock, std::chrono::duration<long, std::ratio<1l, 1000000000l> > >&> (this=0x7ffa178c8210, this@entry=0x7ffa178ae700) at /usr/local/include/c++/7.3.0/bits/vector.tcc:105

No locals.

#7  cb::tracing::Tracer::begin (this=this@entry=0x7ffa178c8210, tracecode=<optimized out>, startTime=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/tracing/tracer.cc:35

No locals.

#8  0x00007ffa1f5a7bef in TracerStopwatch::~TracerStopwatch (this=0x7ffa1686dc28, __in_chrg=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/trace_helpers.h:87

        guard = {engine = 0x7ffa103ff000}

        tracer = <optimized out>

        spanId = <optimized out>

        this = 0x7ffa1686dc28

        __in_chrg = <optimized out>

        guard = <optimized out>

        tracer = <optimized out>

        spanId = <optimized out>

#9  0x00007ffa1f5cb756 in ScopeTimer2<MicrosecondStopwatch<Hdr1sfMicroSecHistogram>, TracerStopwatch>::~ScopeTimer2 (this=0x7ffa1686dc10, __in_chrg=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/include/platform/scope_timer.h:99

No locals.

#10 EventuallyPersistentEngine::storeIfInner(void const*, Item&, unsigned long, ENGINE_STORE_OPERATION, std::function<cb::StoreIfStatus (std::optional<item_info> const&, cb::vbucket_info)> const&, bool) (this=this@entry=0x7ffa103ff000, cookie=cookie@entry=0x7ffa178c8200, item=..., cas=cas@entry=0, operation=operation@entry=OPERATION_ADD, predicate=...,

    preserveTtl=false) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:2557

        timer = {startTime = {__d = {__r = 437752352197214}}, listener1 = {histogram = @0x7ffa103ff450, startTime = {__d = {__r = 437752352197214}}}, listener2 = {traceable = 0x7ffa178c8200, code = cb::tracing::Code::Store, startTime = {__d = {__r = 437752352197214}}, stopTime = {__d = {__r = 437752352297701}}}}

        status = <optimized out>

#11 0x00007ffa1f5cbb28 in EventuallyPersistentEngine::store_if(gsl::not_null<void const*>, gsl::not_null<void*>, unsigned long, ENGINE_STORE_OPERATION, std::function<cb::StoreIfStatus (std::optional<item_info> const&, cb::vbucket_info)> const&, std::optional<cb::durability::Requirements> const&, DocumentState, bool) (this=0x7ffa103ff000, cookie=..., itm=...,

    cas=0, operation=OPERATION_ADD, predicate=..., durability=..., document_state=(DocumentState::Alive | unknown: -256), preserveTtl=false) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:448

        item = <optimized out>

#12 0x0000000000452b5c in bucket_store_if(Cookie&, gsl::not_null<void*>, unsigned long, ENGINE_STORE_OPERATION, std::function<cb::StoreIfStatus (std::optional<item_info> const&, cb::vbucket_info)>, std::optional<cb::durability::Requirements>, DocumentState, bool) (cookie=..., item_=..., cas=0, operation=OPERATION_ADD, predicate=..., durability=...,

    document_state=(DocumentState::Alive | unknown: -256), preserveTtl=false) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:145

        c = <optimized out>

        ret = <optimized out>

#13 0x00000000004c4ce7 in MutationCommandContext::storeItem (this=this@entry=0x7ffa17a1f300) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/mutation_context.cc:294

        request = <optimized out>

        ret = <optimized out>

#14 0x00000000004c5ab8 in MutationCommandContext::step (this=0x7ffa17a1f300) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/mutation_context.cc:60

        ret = ENGINE_SUCCESS

#15 0x00000000004cf250 in SteppableCommandContext::drive (this=this@entry=0x7ffa17a1f300) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/steppable_command_context.cc:37

        ret = ENGINE_SUCCESS

        remapErr = <optimized out>

#16 0x00000000004a890d in add_set_replace_executor (store_op=OPERATION_ADD, cookie=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mcbp_executors.cc:163

No locals.

#17 add_executor (cookie=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mcbp_executors.cc:167

No locals.

#18 0x00000000004a9eb6 in std::function<void (Cookie&)>::operator()(Cookie&) const (__args#0=..., this=<optimized out>) at /usr/local/include/c++/7.3.0/bits/std_function.h:706

No locals.

#19 execute_client_request_packet (cookie=..., request=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mcbp_executors.cc:897

        c = <optimized out>

        privilegeChains = {commandChains = {_M_elems = {{chain = {<std::_Vector_base<Function<cb::rbac::PrivilegeAccess, cb::rbac::PrivilegeAccess::getSuccessValue, Cookie&>, std::allocator<Function<cb::rbac::PrivilegeAccess, cb::rbac::PrivilegeAccess::getSuccessValue, Cookie&> > >> = {

                    _M_impl = {<std::allocator<Function<cb::rbac::PrivilegeAccess, cb::rbac::PrivilegeAccess::getSuccessValue, Cookie&> >> = {<__gnu_cxx::new_allocator<Function<cb::rbac::PrivilegeAccess, cb::rbac::PrivilegeAccess::getSuccessValue, Cookie&> >> = {<No data fields>}, <No data fields>}, _M_start = 0x0, _M_finish = 0x0,

                      _M_end_of_storage = 0x0}}, <No data fields>}} <repeats 256 times>}}}

        opcode = <optimized out>

        res = {status = cb::rbac::PrivilegeAccess::Status::Ok}

#20 0x0000000000498f89 in Cookie::execute (this=this@entry=0x7ffa178c8200) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:153

        header = <error reading variable>

#21 0x0000000000481b5b in Connection::executeCommandPipeline (this=this@entry=0x7ffa1ab87100) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:569

        cookie = {_M_t = {_M_t = {<std::_Tuple_impl<0, Cookie*, std::default_delete<Cookie> >> = {<std::_Tuple_impl<1, std::default_delete<Cookie> >> = {<std::_Head_base<1, std::default_delete<Cookie>, true>> = {<std::default_delete<Cookie>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, Cookie*, false>> = {

                  _M_head_impl = 0x7ffa178c8200}, <No data fields>}, <No data fields>}}}

        drainSize = <optimized out>

        status = <optimized out>

        input = 0x7ffa0c0041e0

        stop = false

        maxActiveCommands = 32

        active = true

        maxSendQueueSize = <optimized out>

#22 0x0000000000485258 in Connection::executeCommandsCallback (this=this@entry=0x7ffa1ab87100) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:638

        start = <optimized out>

        stop = <optimized out>

        ns = <optimized out>

#23 0x0000000000485995 in Connection::rw_callback (ctx=0x7ffa1ab87100) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:762

        instance = <error reading variable>

        thread = <optimized out>

        phosphor_internal_category_enabled_741 = {_M_b = {Segmentation fault (core dumped)

Attachments

- Sort By Name
- Sort By Date
- Ascending
- Descending
- Thumbnails
- List
- Download All

20200512T192607.000-0700.zip
39.87 MB
13/May/20 1:58 AM

Issue Links

duplicates

MB-39991 Race conditions for tracing backing store

Closed

Gerrit Reviews

- Issue Only
- Show All Reviews
- Show Open Reviews
- Show All Issues
- Show Open Issues

No reviews matched the request. Check your Options in the drop-down menu of this sections header.

[Jepsen] Crash in set-kill couchbase bucket test

Details

Description

Attachments

Attachments

Issue Links

Gerrit Reviews

Activity

People

Dates

Gerrit Reviews

PagerDuty