Details
-
Bug
-
Resolution: Fixed
-
Major
-
Cheshire-Cat
-
Triaged
-
1
-
Unknown
Description
As seem during CV for an unrelated change, there is a data race on a number of Stream member variables between PassiveStream::reconnectStream() (as part of rollback) and
Stream::addStats().
Backtrace of the first variable, the others are essentially the same:
Running [0056/0099]: test full rollback on consumer...==================
|
WARNING: ThreadSanitizer: data race (pid=43827)
|
Write of size 8 at 0x7b540001ef60 by thread T23:
|
#0 PassiveStream::reconnectStream(std::shared_ptr<VBucket>&, unsigned int, unsigned long) ../kv_engine/engines/ep/src/dcp/passive_stream.cc:240 (libep.so+0x00000012eed6)
|
#1 DcpConsumer::doRollback(unsigned int, Vbid, unsigned long) ../kv_engine/engines/ep/src/dcp/consumer.cc:1107 (libep.so+0x0000001116d3)
|
#2 RollbackTask::run() ../kv_engine/engines/ep/src/dcp/consumer.cc:938 (libep.so+0x000000111871)
|
#3 GlobalTask::execute() ../kv_engine/engines/ep/src/globaltask.cc:73 (libep.so+0x0000002363cb)
|
#4 CB3ExecutorThread::run() ../kv_engine/engines/ep/src/cb3_executorthread.cc:174 (libep.so+0x00000009f7a1)
|
#5 launch_executor_thread ../kv_engine/engines/ep/src/cb3_executorthread.cc:34 (libep.so+0x00000009fded)
|
#6 CouchbaseThread::run() ../platform/src/cb_pthreads.cc:58 (libplatform_so.so.0.1.0+0x000000010e1b)
|
#7 platform_thread_wrap ../platform/src/cb_pthreads.cc:71 (libplatform_so.so.0.1.0+0x000000010e1b)
|
#8 <null> <null> (libtsan.so.0+0x000000024feb)
|
|
Previous read of size 8 at 0x7b540001ef60 by thread T22 (mutexes: write M639646524455782464):
|
#0 void StatCollector::addStat<std::basic_string_view<char, std::char_traits<char> >&, unsigned long&>(std::basic_string_view<char, std::char_traits<char> >&, unsigned long&) ../kv_engine/include/statistics/collector.h:343 (libep.so+0x0000000e5e03)
|
#1 void add_casted_stat<unsigned long&>(std::basic_string_view<char, std::char_traits<char> >, unsigned long&, std::function<void (std::basic_string_view<char, std::char_traits<char> >, std::basic_string_view<char, std::char_traits<char> >, gsl::not_null<void const*>)> const&, void const*) ../kv_engine/include/statistics/collector.h:404 (libep.so+0x0000000e5e03)
|
#2 Stream::addStats(std::function<void (std::basic_string_view<char, std::char_traits<char> >, std::basic_string_view<char, std::char_traits<char> >, gsl::not_null<void const*>)> const&, void const*) ../kv_engine/engines/ep/src/dcp/stream.cc:157 (libep.so+0x00000016b1b4)
|
#3 PassiveStream::addStats(std::function<void (std::basic_string_view<char, std::char_traits<char> >, std::basic_string_view<char, std::char_traits<char> >, gsl::not_null<void const*>)> const&, void const*) ../kv_engine/engines/ep/src/dcp/passive_stream.cc:1058 (libep.so+0x000000133a98)
|
#4 DcpConsumer::addStats(std::function<void (std::basic_string_view<char, std::char_traits<char> >, std::basic_string_view<char, std::char_traits<char> >, gsl::not_null<void const*>)> const&, void const*) ../kv_engine/engines/ep/src/dcp/consumer.cc:1140 (libep.so+0x000000115415)
|
#5 ConnStatBuilder::operator()(std::shared_ptr<ConnHandler>) ../kv_engine/engines/ep/src/ep_engine.cc:3784 (libep.so+0x0000001f3f9a)
|
#6 void DcpConnMap::each<ConnStatBuilder&>(ConnStatBuilder&) ../kv_engine/engines/ep/src/dcp/dcpconnmap_impl.h:33 (libep.so+0x0000001f3f9a)
|
#7 EventuallyPersistentEngine::doDcpStats(void const*, std::function<void (std::basic_string_view<char, std::char_traits<char> >, std::basic_string_view<char, std::char_traits<char> >, gsl::not_null<void const*>)> const&, std::basic_string_view<char, std::char_traits<char> >) ../kv_engine/engines/ep/src/ep_engine.cc:3943 (libep.so+0x0000001d9cd0)
|
#8 EventuallyPersistentEngine::getStats(void const*, std::basic_string_view<char, std::char_traits<char> >, std::basic_string_view<char, std::char_traits<char> >, std::function<void (std::basic_string_view<char, std::char_traits<char> >, std::basic_string_view<char, std::char_traits<char> >, gsl::not_null<void const*>)> const&) ../kv_engine/engines/ep/src/ep_engine.cc:4675 (libep.so+0x0000001dd67b)
|
#9 KVBucket::snapshotStats() ../kv_engine/engines/ep/src/kv_bucket.cc:1168 (libep.so+0x000000261e32)
|
#10 StatSnap::run() ../kv_engine/engines/ep/src/tasks.cc:72 (libep.so+0x0000002b76a9)
|
#11 GlobalTask::execute() ../kv_engine/engines/ep/src/globaltask.cc:73 (libep.so+0x0000002363cb)
|
#12 CB3ExecutorThread::run() ../kv_engine/engines/ep/src/cb3_executorthread.cc:174 (libep.so+0x00000009f7a1)
|
#13 launch_executor_thread ../kv_engine/engines/ep/src/cb3_executorthread.cc:34 (libep.so+0x00000009fded)
|
#14 CouchbaseThread::run() ../platform/src/cb_pthreads.cc:58 (libplatform_so.so.0.1.0+0x000000010e1b)
|
#15 platform_thread_wrap ../platform/src/cb_pthreads.cc:71 (libplatform_so.so.0.1.0+0x000000010e1b)
|
#16 <null> <null> (libtsan.so.0+0x000000024feb)
|
|
Location is heap block of size 560 at 0x7b540001ef00 allocated by main thread:
|
#0 malloc <null> (libtsan.so.0+0x0000000274e1)
|
#1 cb::SystemArenaMalloc::malloc(unsigned long) ../platform/src/system_arena_malloc.cc:101 (libplatform_so.so.0.1.0+0x000000024c05)
|
#2 cb::_ArenaMalloc<cb::SystemArenaMalloc>::malloc(unsigned long) ../platform/include/platform/cb_arena_malloc.h:140 (ep_testsuite_dcp+0x000000506360)
|
#3 cb_malloc ../platform/src/cb_malloc_arena.cc:46 (ep_testsuite_dcp+0x000000506360)
|
#4 operator new(unsigned long) ../platform/src/global_new_replacement.cc:71 (ep_testsuite_dcp+0x000000506140)
|
#5 __gnu_cxx::new_allocator<std::_Sp_counted_ptr_inplace<PassiveStream, std::allocator<PassiveStream>, (__gnu_cxx::_Lock_policy)2> >::allocate(unsigned long, void const*) /usr/local/include/c++/7.3.0/ext/new_allocator.h:111 (libep.so+0x000000118384)
|
#6 std::allocator_traits<std::allocator<std::_Sp_counted_ptr_inplace<PassiveStream, std::allocator<PassiveStream>, (__gnu_cxx::_Lock_policy)2> > >::allocate(std::allocator<std::_Sp_counted_ptr_inplace<PassiveStream, std::allocator<PassiveStream>, (__gnu_cxx::_Lock_policy)2> >&, unsigned long) /usr/local/include/c++/7.3.0/bits/alloc_traits.h:436 (libep.so+0x000000118384)
|
#7 std::__allocated_ptr<std::allocator<std::_Sp_counted_ptr_inplace<PassiveStream, std::allocator<PassiveStream>, (__gnu_cxx::_Lock_policy)2> > > std::__allocate_guarded<std::allocator<std::_Sp_counted_ptr_inplace<PassiveStream, std::allocator<PassiveStream>, (__gnu_cxx::_Lock_policy)2> > >(std::allocator<std::_Sp_counted_ptr_inplace<PassiveStream, std::allocator<PassiveStream>, (__gnu_cxx::_Lock_policy)2> >&) /usr/local/include/c++/7.3.0/bits/allocated_ptr.h:104 (libep.so+0x000000118384)
|
#8 std::__shared_count<(__gnu_cxx::_Lock_policy)2>::__shared_count<PassiveStream, std::allocator<PassiveStream>, EventuallyPersistentEngine*, std::shared_ptr<DcpConsumer>&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned int&, unsigned int&, Vbid&, unsigned long&, unsigned long&, unsigned long&, unsigned long&, unsigned long&, unsigned long&, Monotonic<unsigned long, ThrowExceptionPolicy, cb::greater_equal> const&>(std::_Sp_make_shared_tag, PassiveStream*, std::allocator<PassiveStream> const&, EventuallyPersistentEngine*&&, std::shared_ptr<DcpConsumer>&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned int&, unsigned int&, Vbid&, unsigned long&, unsigned long&, unsigned long&, unsigned long&, unsigned long&, unsigned long&, Monotonic<unsigned long, ThrowExceptionPolicy, cb::greater_equal> const&) /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:635 (libep.so+0x00000010cb54)
|
#9 std::__shared_ptr<PassiveStream, (__gnu_cxx::_Lock_policy)2>::__shared_ptr<std::allocator<PassiveStream>, EventuallyPersistentEngine*, std::shared_ptr<DcpConsumer>&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned int&, unsigned int&, Vbid&, unsigned long&, unsigned long&, unsigned long&, unsigned long&, unsigned long&, unsigned long&, Monotonic<unsigned long, ThrowExceptionPolicy, cb::greater_equal> const&>(std::_Sp_make_shared_tag, std::allocator<PassiveStream> const&, EventuallyPersistentEngine*&&, std::shared_ptr<DcpConsumer>&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned int&, unsigned int&, Vbid&, unsigned long&, unsigned long&, unsigned long&, unsigned long&, unsigned long&, unsigned long&, Monotonic<unsigned long, ThrowExceptionPolicy, cb::greater_equal> const&) /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:1295 (libep.so+0x00000010cb54)
|
#10 std::shared_ptr<PassiveStream>::shared_ptr<std::allocator<PassiveStream>, EventuallyPersistentEngine*, std::shared_ptr<DcpConsumer>&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned int&, unsigned int&, Vbid&, unsigned long&, unsigned long&, unsigned long&, unsigned long&, unsigned long&, unsigned long&, Monotonic<unsigned long, ThrowExceptionPolicy, cb::greater_equal> const&>(std::_Sp_make_shared_tag, std::allocator<PassiveStream> const&, EventuallyPersistentEngine*&&, std::shared_ptr<DcpConsumer>&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned int&, unsigned int&, Vbid&, unsigned long&, unsigned long&, unsigned long&, unsigned long&, unsigned long&, unsigned long&, Monotonic<unsigned long, ThrowExceptionPolicy, cb::greater_equal> const&) /usr/local/include/c++/7.3.0/bits/shared_ptr.h:344 (libep.so+0x00000010cb54)
|
#11 std::shared_ptr<PassiveStream> std::allocate_shared<PassiveStream, std::allocator<PassiveStream>, EventuallyPersistentEngine*, std::shared_ptr<DcpConsumer>&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned int&, unsigned int&, Vbid&, unsigned long&, unsigned long&, unsigned long&, unsigned long&, unsigned long&, unsigned long&, Monotonic<unsigned long, ThrowExceptionPolicy, cb::greater_equal> const&>(std::allocator<PassiveStream> const&, EventuallyPersistentEngine*&&, std::shared_ptr<DcpConsumer>&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned int&, unsigned int&, Vbid&, unsigned long&, unsigned long&, unsigned long&, unsigned long&, unsigned long&, unsigned long&, Monotonic<unsigned long, ThrowExceptionPolicy, cb::greater_equal> const&) /usr/local/include/c++/7.3.0/bits/shared_ptr.h:691 (libep.so+0x00000010cb54)
|
#12 std::shared_ptr<PassiveStream> std::make_shared<PassiveStream, EventuallyPersistentEngine*, std::shared_ptr<DcpConsumer>&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned int&, unsigned int&, Vbid&, unsigned long&, unsigned long&, unsigned long&, unsigned long&, unsigned long&, unsigned long&, Monotonic<unsigned long, ThrowExceptionPolicy, cb::greater_equal> const&>(EventuallyPersistentEngine*&&, std::shared_ptr<DcpConsumer>&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned int&, unsigned int&, Vbid&, unsigned long&, unsigned long&, unsigned long&, unsigned long&, unsigned long&, unsigned long&, Monotonic<unsigned long, ThrowExceptionPolicy, cb::greater_equal> const&) /usr/local/include/c++/7.3.0/bits/shared_ptr.h:707 (libep.so+0x00000010cb54)
|
#13 DcpConsumer::makePassiveStream(EventuallyPersistentEngine&, std::shared_ptr<DcpConsumer>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned int, unsigned int, Vbid, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, Monotonic<unsigned long, ThrowExceptionPolicy, cb::greater_equal>) ../kv_engine/engines/ep/src/dcp/consumer.cc:251 (libep.so+0x00000010cb54)
|
#14 DcpConsumer::addStream(unsigned int, Vbid, unsigned int) ../kv_engine/engines/ep/src/dcp/consumer.cc:317 (libep.so+0x00000010f9f7)
|
#15 DcpConnMap::addPassiveStream(ConnHandler&, unsigned int, Vbid, unsigned int) ../kv_engine/engines/ep/src/dcp/dcpconnmap.cc:155 (libep.so+0x00000011eb65)
|
#16 EventuallyPersistentEngine::dcpAddStream(void const*, unsigned int, Vbid, unsigned int) ../kv_engine/engines/ep/src/ep_engine.cc:6298 (libep.so+0x0000001d1cc8)
|
#17 EventuallyPersistentEngine::add_stream(gsl::not_null<void const*>, unsigned int, Vbid, unsigned int) ../kv_engine/engines/ep/src/ep_engine.cc:1360 (libep.so+0x0000001d1d38)
|
#18 ENGINE_ERROR_CODE std::__invoke_impl<ENGINE_ERROR_CODE, ENGINE_ERROR_CODE (DcpIface::*&)(gsl::not_null<void const*>, unsigned int, Vbid, unsigned int), DcpIface*&, void const*&, unsigned int&, Vbid&, unsigned int&>(std::__invoke_memfun_deref, ENGINE_ERROR_CODE (DcpIface::*&)(gsl::not_null<void const*>, unsigned int, Vbid, unsigned int), DcpIface*&, void const*&, unsigned int&, Vbid&, unsigned int&) /usr/local/include/c++/7.3.0/bits/invoke.h:73 (ep_testsuite_dcp+0x00000050e7aa)
|
#19 std::__invoke_result<ENGINE_ERROR_CODE (DcpIface::*&)(gsl::not_null<void const*>, unsigned int, Vbid, unsigned int), DcpIface*&, void const*&, unsigned int&, Vbid&, unsigned int&>::type std::__invoke<ENGINE_ERROR_CODE (DcpIface::*&)(gsl::not_null<void const*>, unsigned int, Vbid, unsigned int), DcpIface*&, void const*&, unsigned int&, Vbid&, unsigned int&>(ENGINE_ERROR_CODE (DcpIface::*&)(gsl::not_null<void const*>, unsigned int, Vbid, unsigned int), DcpIface*&, void const*&, unsigned int&, Vbid&, unsigned int&) /usr/local/include/c++/7.3.0/bits/invoke.h:96 (ep_testsuite_dcp+0x00000050e7aa)
|
#20 ENGINE_ERROR_CODE std::_Bind<ENGINE_ERROR_CODE (DcpIface::*(DcpIface*, void const*, unsigned int, Vbid, unsigned int))(gsl::not_null<void const*>, unsigned int, Vbid, unsigned int)>::__call<ENGINE_ERROR_CODE, , 0ul, 1ul, 2ul, 3ul, 4ul>(std::tuple<>&&, std::_Index_tuple<0ul, 1ul, 2ul, 3ul, 4ul>) /usr/local/include/c++/7.3.0/functional:469 (ep_testsuite_dcp+0x00000050e7aa)
|
#21 ENGINE_ERROR_CODE std::_Bind<ENGINE_ERROR_CODE (DcpIface::*(DcpIface*, void const*, unsigned int, Vbid, unsigned int))(gsl::not_null<void const*>, unsigned int, Vbid, unsigned int)>::operator()<, ENGINE_ERROR_CODE>() /usr/local/include/c++/7.3.0/functional:551 (ep_testsuite_dcp+0x00000050e7aa)
|
#22 std::_Function_handler<ENGINE_ERROR_CODE (), std::_Bind<ENGINE_ERROR_CODE (DcpIface::*(DcpIface*, void const*, unsigned int, Vbid, unsigned int))(gsl::not_null<void const*>, unsigned int, Vbid, unsigned int)> >::_M_invoke(std::_Any_data const&) /usr/local/include/c++/7.3.0/bits/std_function.h:302 (ep_testsuite_dcp+0x00000050e7aa)
|
#23 std::function<ENGINE_ERROR_CODE ()>::operator()() const /usr/local/include/c++/7.3.0/bits/std_function.h:706 (ep_testsuite_dcp+0x00000050ee78)
|
#24 call_engine_and_handle_EWOULDBLOCK ../kv_engine/programs/engine_testapp/mock_engine.cc:60 (ep_testsuite_dcp+0x000000508a9b)
|
#25 MockEngine::add_stream(gsl::not_null<void const*>, unsigned int, Vbid, unsigned int) ../kv_engine/programs/engine_testapp/mock_engine.cc:463 (ep_testsuite_dcp+0x000000508fcb)
|
#26 test_fullrollback_for_consumer ../kv_engine/engines/ep/tests/ep_testsuite_dcp.cc:4593 (ep_testsuite_dcp+0x0000004b804b)
|
#27 execute_test ../kv_engine/programs/engine_testapp/engine_testapp.cc:380 (ep_testsuite_dcp+0x0000004475df)
|
#28 main ../kv_engine/programs/engine_testapp/engine_testapp.cc:631 (ep_testsuite_dcp+0x000000448a91)
|
|
Mutex M639646524455782464 is already destroyed.
|
|
|
SUMMARY: ThreadSanitizer: data race ../kv_engine/engines/ep/src/dcp/passive_stream.cc:240 in PassiveStream::reconnectStream(std::shared_ptr<VBucket>&, unsigned int, unsigned long)
|