Details
-
Bug
-
Status: Closed
-
Critical
-
Resolution: Fixed
-
Cheshire-Cat
-
7.0.0-3216
-
Untriaged
-
1
-
Unknown
Description
On node: 172.23.107.50
Core was generated by `/opt/couchbase/bin/memcached -C /opt/couchbase/var/lib/couchbase/config/memcach'.
|
#0 0x00007f5bbb121074 in DcpProducer::getHighSeqnoOfCollections(Collections::VB::Filter const&, Vbid) const () from /opt/couchbase/bin/../lib/libep.so |
#0 0x00007f5bbb121074 in DcpProducer::getHighSeqnoOfCollections(Collections::VB::Filter const&, Vbid) const () from /opt/couchbase/bin/../lib/libep.so |
#1 0x00007f5bbb122783 in DcpProducer::streamRequest(unsigned int, unsigned int, Vbid, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long*, ENGINE_ERROR_CODE (*)(vbucket_failover_t*, unsigned long, gsl::not_null<void const*>), std::optional<std::basic_string_view<char, std::char_traits<char> > >) () from /opt/couchbase/bin/../lib/libep.so |
#2 0x00007f5bbb174340 in EventuallyPersistentEngine::stream_req(gsl::not_null<void const*>, unsigned int, unsigned int, Vbid, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long*, ENGINE_ERROR_CODE (*)(vbucket_failover_t*, unsigned long, gsl::not_null<void const*>), std::optional<std::basic_string_view<char, std::char_traits<char> > >) () from /opt/couchbase/bin/../lib/libep.so |
#3 0x000000000046fd34 in dcpStreamReq(Cookie&, unsigned int, unsigned int, Vbid, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long*, ENGINE_ERROR_CODE (*)(vbucket_failover_t*, unsigned long, gsl::not_null<void const*>), std::optional<std::basic_string_view<char, std::char_traits<char> > >) () |
#4 0x00000000004decac in dcp_stream_req_executor(Cookie&) () |
#5 0x00000000004c8c76 in execute_client_request_packet(Cookie&, cb::mcbp::Request const&) () |
#6 0x00000000004b7d09 in Cookie::execute() () |
#7 0x00000000004a04fd in Connection::executeCommandPipeline() () |
#8 0x00000000004a3c48 in Connection::executeCommandsCallback() () |
#9 0x00000000004a4475 in Connection::rw_callback(bufferevent*, void*) () |
#10 0x00007f5bb8bd0d8e in bufferevent_run_deferred_callbacks_unlocked () from /opt/couchbase/bin/../lib/libevent_core-2.1.so.7 |
#11 0x00007f5bb8bd9d01 in event_process_active_single_queue.isra.31 () from /opt/couchbase/bin/../lib/libevent_core-2.1.so.7 |
#12 0x00007f5bb8bda54f in event_base_loop () from /opt/couchbase/bin/../lib/libevent_core-2.1.so.7 |
#13 0x0000000000487de9 in worker_libevent(void*) () |
#14 0x00007f5bb9b09947 in platform_thread_wrap(void*) () from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0 |
#15 0x00007f5bb734edd5 in start_thread () from /lib64/libpthread.so.0 |
#16 0x00007f5bb7077ead in epoll_wait () from /lib64/libc.so.6 |
#17 0x0000000000000000 in ?? () |
Memcahced msgs on .50
2020-09-27T00:34:45.132961-07:00 CRITICAL Breakpad caught a crash (Couchbase version 7.0.0-3216). Writing crash dump to /opt/couchbase/var/lib/couchbase/crash/a3494228-cd68-46a0-c9d4fbac-1e6fc1d1.dmp before terminating. |
2020-09-27T00:34:45.133002-07:00 CRITICAL Stack backtrace of crashed thread: |
2020-09-27T00:34:45.133244-07:00 CRITICAL /opt/couchbase/bin/memcached() [0x400000+0x19422d] |
2020-09-27T00:34:45.133261-07:00 CRITICAL /opt/couchbase/bin/memcached(_ZN15google_breakpad16ExceptionHandler12GenerateDumpEPNS0_12CrashContextE+0x3ea) [0x400000+0x1aa5aa] |
2020-09-27T00:34:45.133275-07:00 CRITICAL /opt/couchbase/bin/memcached(_ZN15google_breakpad16ExceptionHandler13SignalHandlerEiP9siginfo_tPv+0xb8) [0x400000+0x1aa8e8] |
2020-09-27T00:34:45.133282-07:00 CRITICAL /lib64/libpthread.so.0() [0x7f5bb7347000+0xf5d0] |
2020-09-27T00:34:45.133293-07:00 CRITICAL /opt/couchbase/bin/../lib/libep.so() [0x7f5bbb031000+0xf0074] |
2020-09-27T00:34:45.133301-07:00 CRITICAL /opt/couchbase/bin/../lib/libep.so() [0x7f5bbb031000+0xf1783] |
2020-09-27T00:34:45.133309-07:00 CRITICAL /opt/couchbase/bin/../lib/libep.so() [0x7f5bbb031000+0x143340] |
2020-09-27T00:34:45.133321-07:00 CRITICAL /opt/couchbase/bin/memcached() [0x400000+0x6fd34] |
2020-09-27T00:34:45.133333-07:00 CRITICAL /opt/couchbase/bin/memcached() [0x400000+0xdecac] |
2020-09-27T00:34:45.133344-07:00 CRITICAL /opt/couchbase/bin/memcached() [0x400000+0xc8c76] |
2020-09-27T00:34:45.133353-07:00 CRITICAL /opt/couchbase/bin/memcached() [0x400000+0xb7d09] |
2020-09-27T00:34:45.133362-07:00 CRITICAL /opt/couchbase/bin/memcached() [0x400000+0xa04fd] |
2020-09-27T00:34:45.133371-07:00 CRITICAL /opt/couchbase/bin/memcached() [0x400000+0xa3c48] |
2020-09-27T00:34:45.133379-07:00 CRITICAL /opt/couchbase/bin/memcached() [0x400000+0xa4475] |
2020-09-27T00:34:45.133387-07:00 CRITICAL /opt/couchbase/bin/../lib/libevent_core-2.1.so.7() [0x7f5bb8bc2000+0xed8e] |
2020-09-27T00:34:45.133422-07:00 CRITICAL /opt/couchbase/bin/../lib/libevent_core-2.1.so.7() [0x7f5bb8bc2000+0x17d01] |
2020-09-27T00:34:45.133427-07:00 CRITICAL /opt/couchbase/bin/../lib/libevent_core-2.1.so.7(event_base_loop+0x3bf) [0x7f5bb8bc2000+0x1854f] |
2020-09-27T00:34:45.133436-07:00 CRITICAL /opt/couchbase/bin/memcached() [0x400000+0x87de9] |
2020-09-27T00:34:45.133492-07:00 CRITICAL /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0() [0x7f5bb9af9000+0x10947] |
2020-09-27T00:34:45.133498-07:00 CRITICAL /lib64/libpthread.so.0() [0x7f5bb7347000+0x7dd5] |
2020-09-27T00:34:45.133534-07:00 CRITICAL /lib64/libc.so.6(clone+0x6d) [0x7f5bb6f7a000+0xfdead] |
QE Test:
<testcase name="cbas.cbas_bucket_operations.CBASBucketOperations.compact_cb_bucket_with_cbas_connected,cluster:D,cb_bucket_name:default,cluster_name:testexec.90600,get-cbcollect-info:False,ini:/tmp/testexec.90600.ini,replicas:0,durability:MAJORITY,log_level:debug,conf_file:conf/cbas/py-cbas-functional-tests.conf,upgrade_version:7.0.0-3216,cbas_dataset_name:default_ds,compression_mode:active,rerun:True,spec:py-cbas-functional-tests,num_nodes:4,logs_folder:/data/workspace/centos-p0-durability-vset00-00-service_cbas_6.5_P1_majority/logs/testrunner-20-Sep-26_23-55-42/test_20,GROUP:P1;durability,cbas_bucket_name:default_bucket,num_items:10000,case_number:20,infra_log_level:critical" time="832.962000132">
|
Attachments
Activity
Looks like the vbucket no longer exists and we're dereferencing it on line 2076.
(gdb) p vb
|
$2 = {
|
<std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>> = {
|
<std::__shared_ptr_access<VBucket, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>},
|
members of std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>:
|
_M_ptr = 0x0,
|
_M_refcount = {
|
_M_pi = 0x0
|
}
|
}, <No data fields>}
|
We can see the stream is vb:938
(gdb) p vbucket
|
$1 = {
|
vbid = 938
|
}
|
Which we can see is being deleted just before the we crash:
2020-09-27T00:34:44.976038-07:00 INFO (default) ~VBucket(): vb:939
|
2020-09-27T00:34:44.976049-07:00 INFO (default) Deletion of vb:939 was completed.
|
2020-09-27T00:34:44.976304-07:00 INFO (default) VBucket::setState: transitioning vb:938 with high seqno:32 from:active to:dead
|
2020-09-27T00:34:44.976328-07:00 WARNING 63: (default) DCP (Producer) eq_dcpq:cbas:Default.Local:default:b641015c82f40c31751554c6b6c7194b:0 - (vb:938) Stream closing, sent until seqno 32 remaining items 0, reason: The stream closed early because the vbucket state changed
|
2020-09-27T00:34:44.976495-07:00 INFO 63: (default) DCP (Producer) eq_dcpq:cbas:Default.Local:default:b641015c82f40c31751554c6b6c7194b:0 - (vb:938) Stream closed, 16 items sent from backfill phase, 0 items sent from memory phase, 32 was last seqno sent
|
2020-09-27T00:34:44.977019-07:00 INFO (default) VBucket::setState: transitioning vb:938 with high seqno:32 from:dead to:dead
|
2020-09-27T00:34:44.977051-07:00 INFO (default) EPVBucket::setupDeferredDeletion(0x7f5bb5ad0080) vb:938, revision:1
|
2020-09-27T00:34:44.977067-07:00 INFO (default) Request for vb:938 deletion is in EWOULDBLOCK until the database file is removed from disk
|
2020-09-27T00:34:45.132961-07:00 CRITICAL Breakpad caught a crash (Couchbase version 7.0.0-3216). Writing crash dump to /opt/couchbase/var/lib/couchbase/crash/a3494228-cd68-46a0-c9d4fbac-1e6fc1d1.dmp before terminating.
|
2020-09-27T00:34:45.133002-07:00 CRITICAL Stack backtrace of crashed thread:
|
Can fix this buy just getting DcpProducer::getHighSeqnoOfCollections() to use the valid VBucketPtr that DcpProducer::streamRequest() has in scope for the life time of the STREAM_REQUEST.
As we can see from the crash dump:
(gdb) f 1
|
#1 0x00007f5bbb122783 in DcpProducer::streamRequest (this=0x7f5ba414ac00, flags=<optimized out>, opaque=2852323328, vbucket=..., start_seqno=<optimized out>, end_seqno=<optimized out>, vbucket_uuid=<optimized out>, snap_start_seqno=<optimized out>, snap_end_seqno=<optimized out>, rollback_seqno=<optimized out>, callback=<optimized out>, json=...)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/producer.cc:419
|
419 in /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/producer.cc
|
(gdb) p vbucket
|
$3 = {
|
vbid = 938
|
}
|
(gdb) p vb
|
$4 = {
|
<std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>> = {
|
<std::__shared_ptr_access<VBucket, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>},
|
members of std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>:
|
_M_ptr = 0x7f5b9c34bc00,
|
_M_refcount = {
|
_M_pi = 0x7f5b9c42c140
|
}
|
}, <No data fields>}
|
Build couchbase-server-7.0.0-3397 contains kv_engine commit c1cc31f with commit message:
MB-41718: Fix crash due to vbucket nullptr dereference
Backtrace with symbols
(gdb) bt#0 DcpProducer::getHighSeqnoOfCollections (this=this@entry=0x7f5ba414ac00, filter=..., vbucket=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/producer.cc:2076#1 0x00007f5bbb122783 in DcpProducer::streamRequest (this=0x7f5ba414ac00, flags=<optimized out>, opaque=2852323328, vbucket=..., start_seqno=<optimized out>, end_seqno=<optimized out>, vbucket_uuid=<optimized out>, snap_start_seqno=<optimized out>, snap_end_seqno=<optimized out>, rollback_seqno=<optimized out>, callback=<optimized out>, json=...)at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/producer.cc:419#2 0x00007f5bbb174340 in EventuallyPersistentEngine::stream_req (this=<optimized out>, cookie=..., flags=0, opaque=2852323328, vbucket=..., startSeqno=32, endSeqno=18446744073709551615, vbucketUuid=72026158530616, snapStartSeqno=0, snapEndSeqno=32, rollbackSeqno=0x7f5bb0e5be78, callback=0x508d40 <add_failover_log(vbucket_failover_t*, unsigned long, gsl::not_null<void const*>)>, json=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:1429#3 0x000000000046fd34 in dcpStreamReq (cookie=..., flags=0, opaque=2852323328, vbucket=..., startSeqno=startSeqno@entry=32, endSeqno=endSeqno@entry=18446744073709551615, vbucketUuid=72026158530616, snapStartSeqno=0, snapEndSeqno=32, rollbackSeqno=0x7f5bb0e5be78,callback=0x508d40 <add_failover_log(vbucket_failover_t*, unsigned long, gsl::not_null<void const*>)>, json=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:721#4 0x00000000004decac in dcp_stream_req_executor (cookie=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/dcp_stream_req_executor.cc:66#5 0x00000000004c8c76 in operator() (__args#0=..., this=<optimized out>) at /usr/local/include/c++/7.3.0/bits/std_function.h:706#6 execute_client_request_packet (cookie=..., request=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mcbp_executors.cc:893#7 0x00000000004b7d09 in Cookie::execute (this=0x7f5b6ab64880) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:157#8 0x00000000004a04fd in Connection::executeCommandPipeline (this=this@entry=0x7f5b9f6f2200) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:563#9 0x00000000004a3c48 in Connection::executeCommandsCallback (this=this@entry=0x7f5b9f6f2200) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:635#10 0x00000000004a4475 in Connection::rw_callback (ctx=0x7f5b9f6f2200) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:762#11 0x00007f5bb8bd0d8e in bufferevent_run_deferred_callbacks_unlocked (cb=<optimized out>, arg=0x7f5b6ab64600) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/bufferevent.c:208#12 0x00007f5bb8bd9d01 in event_process_active_single_queue (base=base@entry=0x7f5bb5a35600, max_to_process=max_to_process@entry=2147483647, endtime=endtime@entry=0x0, activeq=<optimized out>) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1726#13 0x00007f5bb8bda54f in event_process_active (base=0x7f5bb5a35600) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1789#14 event_base_loop (base=0x7f5bb5a35600, flags=flags@entry=0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:2012#15 0x0000000000487de9 in worker_libevent (arg=0x7f5bb5a49aa8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:187#16 0x00007f5bb9b09947 in run (this=0x7f5bb5aa2ff0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58#17 platform_thread_wrap (arg=0x7f5bb5aa2ff0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71#18 0x00007f5bb734edd5 in start_thread (arg=0x7f5bb0e75700) at pthread_create.c:307#19 0x00007f5bb7077ead in __libc_ifunc_impl_list (name=<optimized out>, array=0x7f5bb0e75700, max=<optimized out>) at ../sysdeps/x86_64/multiarch/ifunc-impl-list.c:338#20 0x0000000000000000 in ?? ()Code:
2067std::optional<uint64_t> DcpProducer::getHighSeqnoOfCollections(2068206920702071}20722073uint64_t maxHighSeqno = 0;2074auto vb = engine_.getVBucket(vbucket);20752076auto handle = vb->getManifest().lock(coll.first);20772078logger->warn(207920802081vbucket,2082coll.first,2083coll.second);20842085208620872088}2089auto collHighSeqno = handle.getHighSeqno();2090maxHighSeqno = std::max(maxHighSeqno, collHighSeqno);2091}209220932094}2095