ThreadSanitizer: data race in BasicLinkedList::purgeTombstones

 SUMMARY: ThreadSanitizer: data race tlm/deps/boost.exploded/include/boost/intrusive/detail/list_iterator.hpp:79 in boost::intrusive::list_iterator<boost::intrusive::mhtraits<OrderedStoredValue, boost::intrusive::list_member_hook<>, &OrderedStoredValue::seqno_hook>, false>::operator=(boost::intrusive::list_iterator<boost::intrusive::mhtraits<OrderedStoredValue, boost::intrusive::list_member_hook<>, &OrderedStoredValue::seqno_hook>, false> const&)

 ==================

 ==================

 WARNING: ThreadSanitizer: data race (pid=17198)

   Read of size 8 at 0x7b4c00002d40 by thread T2 (mutexes: read M666668208119350224, write M838931151564315824, write M830768325700119856, write M825420198063385568):

     #0 boost::intrusive::list_iterator<boost::intrusive::mhtraits<OrderedStoredValue, boost::intrusive::list_member_hook<>, &OrderedStoredValue::seqno_hook>, false>::pointed_node() const <null> (libep.so+0x0000003c5dd5)

     #1 boost::intrusive::operator==(boost::intrusive::list_iterator<boost::intrusive::mhtraits<OrderedStoredValue, boost::intrusive::list_member_hook<>, &OrderedStoredValue::seqno_hook>, false> const&, boost::intrusive::list_iterator<boost::intrusive::mhtraits<OrderedStoredValue, boost::intrusive::list_member_hook<>, &OrderedStoredValue::seqno_hook>, false> const&) tlm/deps/boost.exploded/include/boost/intrusive/detail/list_iterator.hpp:119 (libep.so+0x0000003c230b)

     #2 BasicLinkedList::updateListElem(std::lock_guard<std::mutex>&, std::lock_guard<std::mutex>&, OrderedStoredValue&) ../kv_engine/engines/ep/src/linked_list.cc:82 (libep.so+0x0000003bf176)

     #3 EphemeralVBucket::modifySeqList(std::lock_guard<std::mutex>&, std::lock_guard<std::mutex>&, OrderedStoredValue&) ../kv_engine/engines/ep/src/ephemeral_vb.cc:932 (libep.so+0x000000327553)

     #4 EphemeralVBucket::softDeleteStoredValue(HashTable::HashBucketLock const&, StoredValue&, bool, VBQueueItemCtx const&, unsigned long, DeleteSource) ../kv_engine/engines/ep/src/ephemeral_vb.cc:584 (libep.so+0x000000327b0e)

     #5 EphemeralVBucket::pageOut(Collections::VB::ReadHandle const&, HashTable::HashBucketLock const&, StoredValue*&) ../kv_engine/engines/ep/src/ephemeral_vb.cc:121 (libep.so+0x000000325e93)

     #6 PagingVisitor::doEviction(HashTable::HashBucketLock const&, StoredValue*) ../kv_engine/engines/ep/src/paging_visitor.cc:387 (libep.so+0x0000003b8273)

     #7 PagingVisitor::visit(HashTable::HashBucketLock const&, StoredValue&) ../kv_engine/engines/ep/src/paging_visitor.cc:158 (libep.so+0x0000003b8010)

     #8 non-virtual thunk to PagingVisitor::visit(HashTable::HashBucketLock const&, StoredValue&) ../kv_engine/engines/ep/src/paging_visitor.cc (libep.so+0x0000003b8339)

     #9 HashTable::pauseResumeVisit(HashTableVisitor&, HashTable::Position&) ../kv_engine/engines/ep/src/hash_table.cc:1266 (libep.so+0x000000358388)

     #10 HashTable::visit(HashTableVisitor&) ../kv_engine/engines/ep/src/hash_table.cc:1163 (libep.so+0x000000358060)

     #11 PagingVisitor::visitBucket(std::shared_ptr<VBucket> const&) ../kv_engine/engines/ep/src/paging_visitor.cc:251 (libep.so+0x0000003b85aa)

     #12 VBCBAdaptor::run() ../kv_engine/engines/ep/src/kv_bucket.cc:2386 (libep.so+0x000000383856)

     #13 GlobalTask::execute() ../kv_engine/engines/ep/src/globaltask.cc:73 (libep.so+0x000000352f59)

     #14 operator() ../kv_engine/engines/ep/src/folly_executorpool.cc:167 (libep.so+0x000000342c65)

     #15 void folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#1}>(folly::detail::function::Data&) tlm/deps/folly.exploded/include/folly/Function.h:387 (libep.so+0x0000003429b5)

     #16 folly::detail::function::FunctionTraits<void ()>::operator()() folly/Function.h:416 (libep.so+0x0000005a5f20)

     #17 folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) folly/executors/ThreadPoolExecutor.cpp:98 (libep.so+0x0000005a5f20)

     #18 folly::CPUThreadPoolExecutor::threadRun(std::shared_ptr<folly::ThreadPoolExecutor::Thread>) folly/executors/CPUThreadPoolExecutor.cpp:265 (libep.so+0x000000582e00)

     #19 void std::__invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&>(std::__invoke_memfun_deref, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/invoke.h:73 (libep.so+0x0000005ad214)

     #20 std::__invoke_result<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&>::type std::__invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&>(void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/invoke.h:95 (libep.so+0x0000005acfd5)

     #21 void std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::__call<void, , 0ul, 1ul>(std::tuple<>&&, std::_Index_tuple<0ul, 1ul>) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/functional:467 (libep.so+0x0000005acfd5)

     #22 void std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::operator()<, void>() /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/functional:549 (libep.so+0x0000005acfd5)

     #23 void folly::detail::function::FunctionTraits<void ()>::callBig<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) folly/Function.h:401 (libep.so+0x0000005acfd5)

     #24 folly::detail::function::FunctionTraits<void ()>::operator()() tlm/deps/folly.exploded/include/folly/Function.h:416 (libep.so+0x00000033e74c)

     #25 operator() tlm/deps/folly.exploded/include/folly/executors/thread_factory/PriorityThreadFactory.h:54 (libep.so+0x00000033f3dc)

     #26 void folly::detail::function::FunctionTraits<void ()>::callBig<folly::PriorityThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) tlm/deps/folly.exploded/include/folly/Function.h:401 (libep.so+0x00000033f28d)

     #27 folly::detail::function::FunctionTraits<void ()>::operator()() tlm/deps/folly.exploded/include/folly/Function.h:416 (libep.so+0x00000033e74c)

     #28 operator() tlm/deps/folly.exploded/include/folly/executors/thread_factory/NamedThreadFactory.h:40 (libep.so+0x00000033e713)

     #29 void std::__invoke_impl<void, folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}&&) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/invoke.h:60 (libep.so+0x00000033e6bd)

     #30 std::__invoke_result<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>::type std::__invoke<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(std::__invoke_result&&, (folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}&&)...) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/invoke.h:95 (libep.so+0x00000033e62d)

     #31 _ZNSt6thread8_InvokerISt5tupleIJZN5folly18NamedThreadFactory9newThreadEONS2_8FunctionIFvvEEEEUlvE_EEE9_M_invokeIJLm0EEEEDTclsr3stdE8__invokespcl10_S_declvalIXT_EEEEESt12_Index_tupleIJXspT_EEE /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/thread:234 (libep.so+0x00000033e5f5)

     #32 std::thread::_Invoker<std::tuple<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}> >::operator()() /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/thread:243 (libep.so+0x00000033e5b5)

     #33 std::thread::_State_impl<std::thread::_Invoker<std::tuple<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}> > >::_M_run() /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/thread:186 (libep.so+0x00000033e439)

     #34 execute_native_thread_routine /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/src/c++11/thread.cc:83 (libstdc++.so.6+0x0000000c1d3e)

   Previous write of size 8 at 0x7b4c00002d40 by thread T30:

     #0 boost::intrusive::list_iterator<boost::intrusive::mhtraits<OrderedStoredValue, boost::intrusive::list_member_hook<>, &OrderedStoredValue::seqno_hook>, false>::operator=(boost::intrusive::list_iterator<boost::intrusive::mhtraits<OrderedStoredValue, boost::intrusive::list_member_hook<>, &OrderedStoredValue::seqno_hook>, false> const&) tlm/deps/boost.exploded/include/boost/intrusive/detail/list_iterator.hpp:79 (libep.so+0x0000003c2436)

     #1 BasicLinkedList::purgeTombstones(long, std::function<bool (DocKey const&, long)>, std::function<bool ()>) ../kv_engine/engines/ep/src/linked_list.cc:360 (libep.so+0x0000003c03da)

     #2 EphemeralVBucket::purgeStaleItems(std::function<bool ()>) ../kv_engine/engines/ep/src/ephemeral_vb.cc:391 (libep.so+0x000000326f56)

     #3 EphemeralVBucket::StaleItemDeleter::visit(VBucket&) ../kv_engine/engines/ep/src/ephemeral_tombstone_purger.cc:211 (libep.so+0x000000324a5e)

     #4 KVBucket::pauseResumeVisit(PauseResumeVBVisitor&, KVBucketIface::Position&) ../kv_engine/engines/ep/src/kv_bucket.cc:2322 (libep.so+0x0000003831e8)

     #5 EphTombstoneStaleItemDeleter::run() ../kv_engine/engines/ep/src/ephemeral_tombstone_purger.cc:280 (libep.so+0x0000003211dc)

     #6 GlobalTask::execute() ../kv_engine/engines/ep/src/globaltask.cc:73 (libep.so+0x000000352f59)

     #7 operator() ../kv_engine/engines/ep/src/folly_executorpool.cc:167 (libep.so+0x000000342c65)

     #8 void folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#1}>(folly::detail::function::Data&) tlm/deps/folly.exploded/include/folly/Function.h:387 (libep.so+0x0000003429b5)

     #9 folly::detail::function::FunctionTraits<void ()>::operator()() folly/Function.h:416 (libep.so+0x0000005a5f20)

     #10 folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) folly/executors/ThreadPoolExecutor.cpp:98 (libep.so+0x0000005a5f20)

     #11 folly::CPUThreadPoolExecutor::threadRun(std::shared_ptr<folly::ThreadPoolExecutor::Thread>) folly/executors/CPUThreadPoolExecutor.cpp:265 (libep.so+0x000000582e00)

     #12 void std::__invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&>(std::__invoke_memfun_deref, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/invoke.h:73 (libep.so+0x0000005ad214)

     #13 std::__invoke_result<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&>::type std::__invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&>(void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/invoke.h:95 (libep.so+0x0000005acfd5)

     #14 void std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::__call<void, , 0ul, 1ul>(std::tuple<>&&, std::_Index_tuple<0ul, 1ul>) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/functional:467 (libep.so+0x0000005acfd5)

     #15 void std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::operator()<, void>() /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/functional:549 (libep.so+0x0000005acfd5)

     #16 void folly::detail::function::FunctionTraits<void ()>::callBig<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) folly/Function.h:401 (libep.so+0x0000005acfd5)

     #17 folly::detail::function::FunctionTraits<void ()>::operator()() tlm/deps/folly.exploded/include/folly/Function.h:416 (libep.so+0x00000033e74c)

     #18 operator() tlm/deps/folly.exploded/include/folly/executors/thread_factory/PriorityThreadFactory.h:54 (libep.so+0x00000033f3dc)

     #19 void folly::detail::function::FunctionTraits<void ()>::callBig<folly::PriorityThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) tlm/deps/folly.exploded/include/folly/Function.h:401 (libep.so+0x00000033f28d)

     #20 folly::detail::function::FunctionTraits<void ()>::operator()() tlm/deps/folly.exploded/include/folly/Function.h:416 (libep.so+0x00000033e74c)

     #21 operator() tlm/deps/folly.exploded/include/folly/executors/thread_factory/NamedThreadFactory.h:40 (libep.so+0x00000033e713)

     #22 void std::__invoke_impl<void, folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}&&) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/invoke.h:60 (libep.so+0x00000033e6bd)

     #23 std::__invoke_result<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>::type std::__invoke<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(std::__invoke_result&&, (folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}&&)...) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/invoke.h:95 (libep.so+0x00000033e62d)

     #24 _ZNSt6thread8_InvokerISt5tupleIJZN5folly18NamedThreadFactory9newThreadEONS2_8FunctionIFvvEEEEUlvE_EEE9_M_invokeIJLm0EEEEDTclsr3stdE8__invokespcl10_S_declvalIXT_EEEEESt12_Index_tupleIJXspT_EEE /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/thread:234 (libep.so+0x00000033e5f5)

     #25 std::thread::_Invoker<std::tuple<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}> >::operator()() /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/thread:243 (libep.so+0x00000033e5b5)

     #26 std::thread::_State_impl<std::thread::_Invoker<std::tuple<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}> > >::_M_run() /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/thread:186 (libep.so+0x00000033e439)

     #27 execute_native_thread_routine /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/src/c++11/thread.cc:83 (libstdc++.so.6+0x0000000c1d3e)

354

            if (lastLockedSeqno != purgeUpToSeqno) {

355

                //  have reached the end of the locked range, but the original

356

                //  requested end was higher i.e., the range lock was partial. Pause

357

                //  so next time purge is attempted it will resume from here (the

358

                //  range lock "blocking" part of the requested seqno range may have

359

                //  moved/gone)

360

                pausedPurgePoint = it;

361

            }