Details
-
Bug
-
Resolution: Fixed
-
Major
-
6.5.1, 6.6.0, 6.6.1, 6.6.2, 6.5.2, 6.5.0, Cheshire-Cat
-
Triaged
-
1
-
Unknown
-
KV-Engine 2021-Jan
Description
Following data race reported for an unrelated patch:
SUMMARY: ThreadSanitizer: data race tlm/deps/boost.exploded/include/boost/intrusive/detail/list_iterator.hpp:79 in boost::intrusive::list_iterator<boost::intrusive::mhtraits<OrderedStoredValue, boost::intrusive::list_member_hook<>, &OrderedStoredValue::seqno_hook>, false>::operator=(boost::intrusive::list_iterator<boost::intrusive::mhtraits<OrderedStoredValue, boost::intrusive::list_member_hook<>, &OrderedStoredValue::seqno_hook>, false> const&)
|
==================
|
==================
|
WARNING: ThreadSanitizer: data race (pid=17198)
|
Read of size 8 at 0x7b4c00002d40 by thread T2 (mutexes: read M666668208119350224, write M838931151564315824, write M830768325700119856, write M825420198063385568):
|
#0 boost::intrusive::list_iterator<boost::intrusive::mhtraits<OrderedStoredValue, boost::intrusive::list_member_hook<>, &OrderedStoredValue::seqno_hook>, false>::pointed_node() const <null> (libep.so+0x0000003c5dd5)
|
#1 boost::intrusive::operator==(boost::intrusive::list_iterator<boost::intrusive::mhtraits<OrderedStoredValue, boost::intrusive::list_member_hook<>, &OrderedStoredValue::seqno_hook>, false> const&, boost::intrusive::list_iterator<boost::intrusive::mhtraits<OrderedStoredValue, boost::intrusive::list_member_hook<>, &OrderedStoredValue::seqno_hook>, false> const&) tlm/deps/boost.exploded/include/boost/intrusive/detail/list_iterator.hpp:119 (libep.so+0x0000003c230b)
|
#2 BasicLinkedList::updateListElem(std::lock_guard<std::mutex>&, std::lock_guard<std::mutex>&, OrderedStoredValue&) ../kv_engine/engines/ep/src/linked_list.cc:82 (libep.so+0x0000003bf176)
|
#3 EphemeralVBucket::modifySeqList(std::lock_guard<std::mutex>&, std::lock_guard<std::mutex>&, OrderedStoredValue&) ../kv_engine/engines/ep/src/ephemeral_vb.cc:932 (libep.so+0x000000327553)
|
#4 EphemeralVBucket::softDeleteStoredValue(HashTable::HashBucketLock const&, StoredValue&, bool, VBQueueItemCtx const&, unsigned long, DeleteSource) ../kv_engine/engines/ep/src/ephemeral_vb.cc:584 (libep.so+0x000000327b0e)
|
#5 EphemeralVBucket::pageOut(Collections::VB::ReadHandle const&, HashTable::HashBucketLock const&, StoredValue*&) ../kv_engine/engines/ep/src/ephemeral_vb.cc:121 (libep.so+0x000000325e93)
|
#6 PagingVisitor::doEviction(HashTable::HashBucketLock const&, StoredValue*) ../kv_engine/engines/ep/src/paging_visitor.cc:387 (libep.so+0x0000003b8273)
|
#7 PagingVisitor::visit(HashTable::HashBucketLock const&, StoredValue&) ../kv_engine/engines/ep/src/paging_visitor.cc:158 (libep.so+0x0000003b8010)
|
#8 non-virtual thunk to PagingVisitor::visit(HashTable::HashBucketLock const&, StoredValue&) ../kv_engine/engines/ep/src/paging_visitor.cc (libep.so+0x0000003b8339)
|
#9 HashTable::pauseResumeVisit(HashTableVisitor&, HashTable::Position&) ../kv_engine/engines/ep/src/hash_table.cc:1266 (libep.so+0x000000358388)
|
#10 HashTable::visit(HashTableVisitor&) ../kv_engine/engines/ep/src/hash_table.cc:1163 (libep.so+0x000000358060)
|
#11 PagingVisitor::visitBucket(std::shared_ptr<VBucket> const&) ../kv_engine/engines/ep/src/paging_visitor.cc:251 (libep.so+0x0000003b85aa)
|
#12 VBCBAdaptor::run() ../kv_engine/engines/ep/src/kv_bucket.cc:2386 (libep.so+0x000000383856)
|
#13 GlobalTask::execute() ../kv_engine/engines/ep/src/globaltask.cc:73 (libep.so+0x000000352f59)
|
#14 operator() ../kv_engine/engines/ep/src/folly_executorpool.cc:167 (libep.so+0x000000342c65)
|
#15 void folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#1}>(folly::detail::function::Data&) tlm/deps/folly.exploded/include/folly/Function.h:387 (libep.so+0x0000003429b5)
|
#16 folly::detail::function::FunctionTraits<void ()>::operator()() folly/Function.h:416 (libep.so+0x0000005a5f20)
|
#17 folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) folly/executors/ThreadPoolExecutor.cpp:98 (libep.so+0x0000005a5f20)
|
#18 folly::CPUThreadPoolExecutor::threadRun(std::shared_ptr<folly::ThreadPoolExecutor::Thread>) folly/executors/CPUThreadPoolExecutor.cpp:265 (libep.so+0x000000582e00)
|
#19 void std::__invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&>(std::__invoke_memfun_deref, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/invoke.h:73 (libep.so+0x0000005ad214)
|
#20 std::__invoke_result<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&>::type std::__invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&>(void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/invoke.h:95 (libep.so+0x0000005acfd5)
|
#21 void std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::__call<void, , 0ul, 1ul>(std::tuple<>&&, std::_Index_tuple<0ul, 1ul>) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/functional:467 (libep.so+0x0000005acfd5)
|
#22 void std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::operator()<, void>() /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/functional:549 (libep.so+0x0000005acfd5)
|
#23 void folly::detail::function::FunctionTraits<void ()>::callBig<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) folly/Function.h:401 (libep.so+0x0000005acfd5)
|
#24 folly::detail::function::FunctionTraits<void ()>::operator()() tlm/deps/folly.exploded/include/folly/Function.h:416 (libep.so+0x00000033e74c)
|
#25 operator() tlm/deps/folly.exploded/include/folly/executors/thread_factory/PriorityThreadFactory.h:54 (libep.so+0x00000033f3dc)
|
#26 void folly::detail::function::FunctionTraits<void ()>::callBig<folly::PriorityThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) tlm/deps/folly.exploded/include/folly/Function.h:401 (libep.so+0x00000033f28d)
|
#27 folly::detail::function::FunctionTraits<void ()>::operator()() tlm/deps/folly.exploded/include/folly/Function.h:416 (libep.so+0x00000033e74c)
|
#28 operator() tlm/deps/folly.exploded/include/folly/executors/thread_factory/NamedThreadFactory.h:40 (libep.so+0x00000033e713)
|
#29 void std::__invoke_impl<void, folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}&&) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/invoke.h:60 (libep.so+0x00000033e6bd)
|
#30 std::__invoke_result<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>::type std::__invoke<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(std::__invoke_result&&, (folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}&&)...) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/invoke.h:95 (libep.so+0x00000033e62d)
|
#31 _ZNSt6thread8_InvokerISt5tupleIJZN5folly18NamedThreadFactory9newThreadEONS2_8FunctionIFvvEEEEUlvE_EEE9_M_invokeIJLm0EEEEDTclsr3stdE8__invokespcl10_S_declvalIXT_EEEEESt12_Index_tupleIJXspT_EEE /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/thread:234 (libep.so+0x00000033e5f5)
|
#32 std::thread::_Invoker<std::tuple<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}> >::operator()() /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/thread:243 (libep.so+0x00000033e5b5)
|
#33 std::thread::_State_impl<std::thread::_Invoker<std::tuple<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}> > >::_M_run() /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/thread:186 (libep.so+0x00000033e439)
|
#34 execute_native_thread_routine /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/src/c++11/thread.cc:83 (libstdc++.so.6+0x0000000c1d3e)
|
|
Previous write of size 8 at 0x7b4c00002d40 by thread T30:
|
#0 boost::intrusive::list_iterator<boost::intrusive::mhtraits<OrderedStoredValue, boost::intrusive::list_member_hook<>, &OrderedStoredValue::seqno_hook>, false>::operator=(boost::intrusive::list_iterator<boost::intrusive::mhtraits<OrderedStoredValue, boost::intrusive::list_member_hook<>, &OrderedStoredValue::seqno_hook>, false> const&) tlm/deps/boost.exploded/include/boost/intrusive/detail/list_iterator.hpp:79 (libep.so+0x0000003c2436)
|
#1 BasicLinkedList::purgeTombstones(long, std::function<bool (DocKey const&, long)>, std::function<bool ()>) ../kv_engine/engines/ep/src/linked_list.cc:360 (libep.so+0x0000003c03da)
|
#2 EphemeralVBucket::purgeStaleItems(std::function<bool ()>) ../kv_engine/engines/ep/src/ephemeral_vb.cc:391 (libep.so+0x000000326f56)
|
#3 EphemeralVBucket::StaleItemDeleter::visit(VBucket&) ../kv_engine/engines/ep/src/ephemeral_tombstone_purger.cc:211 (libep.so+0x000000324a5e)
|
#4 KVBucket::pauseResumeVisit(PauseResumeVBVisitor&, KVBucketIface::Position&) ../kv_engine/engines/ep/src/kv_bucket.cc:2322 (libep.so+0x0000003831e8)
|
#5 EphTombstoneStaleItemDeleter::run() ../kv_engine/engines/ep/src/ephemeral_tombstone_purger.cc:280 (libep.so+0x0000003211dc)
|
#6 GlobalTask::execute() ../kv_engine/engines/ep/src/globaltask.cc:73 (libep.so+0x000000352f59)
|
#7 operator() ../kv_engine/engines/ep/src/folly_executorpool.cc:167 (libep.so+0x000000342c65)
|
#8 void folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#1}>(folly::detail::function::Data&) tlm/deps/folly.exploded/include/folly/Function.h:387 (libep.so+0x0000003429b5)
|
#9 folly::detail::function::FunctionTraits<void ()>::operator()() folly/Function.h:416 (libep.so+0x0000005a5f20)
|
#10 folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) folly/executors/ThreadPoolExecutor.cpp:98 (libep.so+0x0000005a5f20)
|
#11 folly::CPUThreadPoolExecutor::threadRun(std::shared_ptr<folly::ThreadPoolExecutor::Thread>) folly/executors/CPUThreadPoolExecutor.cpp:265 (libep.so+0x000000582e00)
|
#12 void std::__invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&>(std::__invoke_memfun_deref, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/invoke.h:73 (libep.so+0x0000005ad214)
|
#13 std::__invoke_result<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&>::type std::__invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&>(void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/invoke.h:95 (libep.so+0x0000005acfd5)
|
#14 void std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::__call<void, , 0ul, 1ul>(std::tuple<>&&, std::_Index_tuple<0ul, 1ul>) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/functional:467 (libep.so+0x0000005acfd5)
|
#15 void std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::operator()<, void>() /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/functional:549 (libep.so+0x0000005acfd5)
|
#16 void folly::detail::function::FunctionTraits<void ()>::callBig<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) folly/Function.h:401 (libep.so+0x0000005acfd5)
|
#17 folly::detail::function::FunctionTraits<void ()>::operator()() tlm/deps/folly.exploded/include/folly/Function.h:416 (libep.so+0x00000033e74c)
|
#18 operator() tlm/deps/folly.exploded/include/folly/executors/thread_factory/PriorityThreadFactory.h:54 (libep.so+0x00000033f3dc)
|
#19 void folly::detail::function::FunctionTraits<void ()>::callBig<folly::PriorityThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) tlm/deps/folly.exploded/include/folly/Function.h:401 (libep.so+0x00000033f28d)
|
#20 folly::detail::function::FunctionTraits<void ()>::operator()() tlm/deps/folly.exploded/include/folly/Function.h:416 (libep.so+0x00000033e74c)
|
#21 operator() tlm/deps/folly.exploded/include/folly/executors/thread_factory/NamedThreadFactory.h:40 (libep.so+0x00000033e713)
|
#22 void std::__invoke_impl<void, folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}&&) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/invoke.h:60 (libep.so+0x00000033e6bd)
|
#23 std::__invoke_result<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>::type std::__invoke<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(std::__invoke_result&&, (folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}&&)...) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/invoke.h:95 (libep.so+0x00000033e62d)
|
#24 _ZNSt6thread8_InvokerISt5tupleIJZN5folly18NamedThreadFactory9newThreadEONS2_8FunctionIFvvEEEEUlvE_EEE9_M_invokeIJLm0EEEEDTclsr3stdE8__invokespcl10_S_declvalIXT_EEEEESt12_Index_tupleIJXspT_EEE /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/thread:234 (libep.so+0x00000033e5f5)
|
#25 std::thread::_Invoker<std::tuple<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}> >::operator()() /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/thread:243 (libep.so+0x00000033e5b5)
|
#26 std::thread::_State_impl<std::thread::_Invoker<std::tuple<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}> > >::_M_run() /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/thread:186 (libep.so+0x00000033e439)
|
#27 execute_native_thread_routine /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/src/c++11/thread.cc:83 (libstdc++.so.6+0x0000000c1d3e)
|
The following assignment is not done under the ListWriteLock, and may race with updateListElem
kv_engine/engines/ep/src/linked_list.cc BasicLinkedList::purgeTombstones |
|
354
|
if (lastLockedSeqno != purgeUpToSeqno) {
|
355
|
// have reached the end of the locked range, but the original
|
356
|
// requested end was higher i.e., the range lock was partial. Pause
|
357
|
// so next time purge is attempted it will resume from here (the
|
358
|
// range lock "blocking" part of the requested seqno range may have
|
359
|
// moved/gone)
|
360
|
pausedPurgePoint = it;
|
361
|
}
|
Attachments
Issue Links
- Clones
-
MB-41406 ThreadSanitizer: data race in BasicLinkedList::purgeTombstones
-
- Closed
-
For Gerrit Dashboard: MB-43717 | ||||||
---|---|---|---|---|---|---|
# | Subject | Branch | Project | Status | CR | V |
143784,2 | MB-43717: Fix race in BasicLinkedList::purgeTombstones | master | kv_engine | Status: MERGED | +2 | +1 |
143877,2 | MB-43717: [BP] Fix race in BasicLinkedList::purgeTombstones | mad-hatter | kv_engine | Status: MERGED | +2 | +1 |
145105,1 | Merge branch 'mad-hatter' into master | master | kv_engine | Status: MERGED | +2 | +1 |