Data race on CheckpointCursor in StreamTest.BackfillOnly/ephemeral

15:20:02  [ RUN      ] PersistentAndEphemeral/StreamTest.BackfillOnly/ephemeral

15:20:02  ==================

15:20:02  WARNING: ThreadSanitizer: data race (pid=22881)

15:20:02    Write of size 8 at 0x7b180003fa60 by thread T54 (mutexes: read M762651381336167400, read M761525309630578840, write M796428430081043512, write M793050678820918888, write M793332050718360480, write M757021795902905520):

15:20:02      #0 boost::intrusive::list_iterator<boost::intrusive::bhtraits<boost::container::dtl::list_node<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, void*>, boost::intrusive::list_node_traits<void*>, (boost::intrusive::link_mode_type)0, boost::intrusive::dft_tag, 1u>, false>::operator--() tlm/deps/boost.exploded/include/boost/intrusive/detail/list_iterator.hpp:107 (ep-engine_ep_unit_tests+0x99f103)

15:20:02      #1 boost::container::dtl::iterator_from_iiterator<boost::intrusive::list_iterator<boost::intrusive::bhtraits<boost::container::dtl::list_node<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, void*>, boost::intrusive::list_node_traits<void*>, (boost::intrusive::link_mode_type)0, boost::intrusive::dft_tag, 1u>, false>, false>::operator--() tlm/deps/boost.exploded/include/boost/container/detail/iterators.hpp:853 (ep-engine_ep_unit_tests+0x99f0c5)

15:20:02      #2 CheckpointIterator<boost::container::list<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, MemoryTrackingAllocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, cb::NonNegativeCounter<unsigned long, cb::DefaultUnderflowPolicy> > > >::moveBackward() ../kv_engine/engines/ep/src/checkpoint_iterator.h:177 (ep-engine_ep_unit_tests+0x99f049)

15:20:02      #3 CheckpointIterator<boost::container::list<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, MemoryTrackingAllocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, cb::NonNegativeCounter<unsigned long, cb::DefaultUnderflowPolicy> > > >::operator--() ../kv_engine/engines/ep/src/checkpoint_iterator.h (ep-engine_ep_unit_tests+0x999f67)

15:20:02      #4 CheckpointCursor::decrPos() /home/couchbase/jenkins/workspace/kv_engine.threadsanitizer_neo/kv_engine/engines/ep/src/checkpoint_cursor.cc:83 (ep-engine_ep_unit_tests+0x989443)

15:20:02      #5 Checkpoint::queueDirty(SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> > const&) /home/couchbase/jenkins/workspace/kv_engine.threadsanitizer_neo/kv_engine/engines/ep/src/checkpoint.cc:255 (ep-engine_ep_unit_tests+0x9b8c71)

15:20:02      #6 CheckpointManager::queueDirty(SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >&, GenerateBySeqno, GenerateCas, PreLinkDocumentContext*, std::function<void (long)>) /home/couchbase/jenkins/workspace/kv_engine.threadsanitizer_neo/kv_engine/engines/ep/src/checkpoint_manager.cc:915 (ep-engine_ep_unit_tests+0x98e5c9)

15:20:02      #7 VBucket::queueItem(SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >&, VBQueueItemCtx const&) /home/couchbase/jenkins/workspace/kv_engine.threadsanitizer_neo/kv_engine/engines/ep/src/vbucket.cc:1304 (ep-engine_ep_unit_tests+0x6bc76e)

15:20:02      #8 VBucket::queueDirty(HashTable::HashBucketLock const&, StoredValue&, VBQueueItemCtx const&) /home/couchbase/jenkins/workspace/kv_engine.threadsanitizer_neo/kv_engine/engines/ep/src/vbucket.cc:1394 (ep-engine_ep_unit_tests+0x6bcbec)

15:20:02      #9 EphemeralVBucket::updateStoredValue(HashTable::HashBucketLock const&, StoredValue&, Item const&, VBQueueItemCtx const&, bool) /home/couchbase/jenkins/workspace/kv_engine.threadsanitizer_neo/kv_engine/engines/ep/src/ephemeral_vb.cc:446 (ep-engine_ep_unit_tests+0x84fa59)

15:20:02      #10 VBucket::processGetAndUpdateTtl(HashTable::HashBucketLock&, StoredValue*, long, Collections::VB::CachingReadHandle const&) /home/couchbase/jenkins/workspace/kv_engine.threadsanitizer_neo/kv_engine/engines/ep/src/vbucket.cc:2780 (ep-engine_ep_unit_tests+0x6c3362)

15:20:02      #11 VBucket::getAndUpdateTtl(CookieIface const*, EventuallyPersistentEngine&, long, Collections::VB::CachingReadHandle const&) /home/couchbase/jenkins/workspace/kv_engine.threadsanitizer_neo/kv_engine/engines/ep/src/vbucket.cc:2818 (ep-engine_ep_unit_tests+0x6c3536)

15:20:02      #12 KVBucket::getAndUpdateTtl(DocKey const&, Vbid, CookieIface const*, long) /home/couchbase/jenkins/workspace/kv_engine.threadsanitizer_neo/kv_engine/engines/ep/src/kv_bucket.cc:1882 (ep-engine_ep_unit_tests+0x77917a)

15:20:02      #13 EventuallyPersistentEngine::getAndTouchInner(CookieIface const*, DocKey const&, Vbid, unsigned int) /home/couchbase/jenkins/workspace/kv_engine.threadsanitizer_neo/kv_engine/engines/ep/src/ep_engine.cc:2443 (ep-engine_ep_unit_tests+0x82a91f)

15:20:02      #14 StreamTest_BackfillOnly_Test::TestBody()::$_3::operator()() const ../kv_engine/engines/ep/tests/module_tests/dcp_stream_test.cc:774 (ep-engine_ep_unit_tests+0x112a30e)

15:20:02      #15 void std::__invoke_impl<void, StreamTest_BackfillOnly_Test::TestBody()::$_3>(std::__invoke_other, StreamTest_BackfillOnly_Test::TestBody()::$_3&&) /opt/gcc-10.2.0/lib/gcc/x86_64-pc-linux-gnu/10.2.0/../../../../include/c++/10.2.0/bits/invoke.h:60 (ep-engine_ep_unit_tests+0x112a1bd)

15:20:02      #16 std::__invoke_result<StreamTest_BackfillOnly_Test::TestBody()::$_3>::type std::__invoke<StreamTest_BackfillOnly_Test::TestBody()::$_3>(StreamTest_BackfillOnly_Test::TestBody()::$_3&&) /opt/gcc-10.2.0/lib/gcc/x86_64-pc-linux-gnu/10.2.0/../../../../include/c++/10.2.0/bits/invoke.h:95 (ep-engine_ep_unit_tests+0x112a12d)

15:20:02      #17 void std::thread::_Invoker<std::tuple<StreamTest_BackfillOnly_Test::TestBody()::$_3> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) /opt/gcc-10.2.0/lib/gcc/x86_64-pc-linux-gnu/10.2.0/../../../../include/c++/10.2.0/thread:264 (ep-engine_ep_unit_tests+0x112a0f5)

15:20:02      #18 std::thread::_Invoker<std::tuple<StreamTest_BackfillOnly_Test::TestBody()::$_3> >::operator()() /opt/gcc-10.2.0/lib/gcc/x86_64-pc-linux-gnu/10.2.0/../../../../include/c++/10.2.0/thread:271 (ep-engine_ep_unit_tests+0x112a0b5)

15:20:02      #19 std::thread::_State_impl<std::thread::_Invoker<std::tuple<StreamTest_BackfillOnly_Test::TestBody()::$_3> > >::_M_run() /opt/gcc-10.2.0/lib/gcc/x86_64-pc-linux-gnu/10.2.0/../../../../include/c++/10.2.0/thread:215 (ep-engine_ep_unit_tests+0x1129fe9)

15:20:02      #20 execute_native_thread_routine /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 (libstdc++.so.6+0xd62ef)

15:20:02  

15:20:02    Previous read of size 8 at 0x7b180003fa60 by thread T49 (mutexes: read M762651381336167400, write M810220635220354168):

15:20:02      #0 boost::intrusive::list_iterator<boost::intrusive::bhtraits<boost::container::dtl::list_node<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, void*>, boost::intrusive::list_node_traits<void*>, (boost::intrusive::link_mode_type)0, boost::intrusive::dft_tag, 1u>, false>::pointed_node() const tlm/deps/boost.exploded/include/boost/intrusive/detail/list_iterator.hpp:82 (ep-engine_ep_unit_tests+0x973fa5)

15:20:02      #1 boost::intrusive::list_iterator<boost::intrusive::bhtraits<boost::container::dtl::list_node<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, void*>, boost::intrusive::list_node_traits<void*>, (boost::intrusive::link_mode_type)0, boost::intrusive::dft_tag, 1u>, false>::list_iterator(boost::intrusive::list_iterator<boost::intrusive::bhtraits<boost::container::dtl::list_node<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, void*>, boost::intrusive::list_node_traits<void*>, (boost::intrusive::link_mode_type)0, boost::intrusive::dft_tag, 1u>, false> const&) tlm/deps/boost.exploded/include/boost/intrusive/detail/list_iterator.hpp:71 (ep-engine_ep_unit_tests+0x973f30)

15:20:02      #2 boost::container::dtl::iterator_from_iiterator<boost::intrusive::list_iterator<boost::intrusive::bhtraits<boost::container::dtl::list_node<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, void*>, boost::intrusive::list_node_traits<void*>, (boost::intrusive::link_mode_type)0, boost::intrusive::dft_tag, 1u>, false>, false>::iterator_from_iiterator(boost::container::dtl::iterator_from_iiterator<boost::intrusive::list_iterator<boost::intrusive::bhtraits<boost::container::dtl::list_node<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, void*>, boost::intrusive::list_node_traits<void*>, (boost::intrusive::link_mode_type)0, boost::intrusive::dft_tag, 1u>, false>, false> const&) tlm/deps/boost.exploded/include/boost/container/detail/iterators.hpp:829 (ep-engine_ep_unit_tests+0x973ed6)

15:20:02      #3 CheckpointIterator<boost::container::list<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, MemoryTrackingAllocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, cb::NonNegativeCounter<unsigned long, cb::DefaultUnderflowPolicy> > > >::CheckpointIterator(CheckpointIterator<boost::container::list<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, MemoryTrackingAllocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, cb::NonNegativeCounter<unsigned long, cb::DefaultUnderflowPolicy> > > > const&) ../kv_engine/engines/ep/src/checkpoint_iterator.h:26 (ep-engine_ep_unit_tests+0x973e9d)

15:20:02      #4 CheckpointCursor::getPos() const ../kv_engine/engines/ep/src/checkpoint_cursor.h:101 (ep-engine_ep_unit_tests+0x96a832)

15:20:02      #5 ActiveStream::registerCursor(CheckpointManager&, unsigned long) /home/couchbase/jenkins/workspace/kv_engine.threadsanitizer_neo/kv_engine/engines/ep/src/dcp/active_stream.cc:271 (ep-engine_ep_unit_tests+0x956d1c)

15:20:02      #6 ActiveStream::markDiskSnapshot(unsigned long, unsigned long, std::optional<unsigned long>, unsigned long, std::optional<unsigned long>, SnapshotType) /home/couchbase/jenkins/workspace/kv_engine.threadsanitizer_neo/kv_engine/engines/ep/src/dcp/active_stream.cc:435 (ep-engine_ep_unit_tests+0x957682)

15:20:02      #7 DCPBackfillMemoryBuffered::create() /home/couchbase/jenkins/workspace/kv_engine.threadsanitizer_neo/kv_engine/engines/ep/src/dcp/backfill_memory.cc:171 (ep-engine_ep_unit_tests+0x9671d6)

15:20:02      #8 DCPBackfillMemoryBuffered::run() /home/couchbase/jenkins/workspace/kv_engine.threadsanitizer_neo/kv_engine/engines/ep/src/dcp/backfill_memory.cc:80 (ep-engine_ep_unit_tests+0x966d18)

15:20:02      #9 BackfillManager::backfill() /home/couchbase/jenkins/workspace/kv_engine.threadsanitizer_neo/kv_engine/engines/ep/src/dcp/backfill-manager.cc:345 (ep-engine_ep_unit_tests+0x96525d)

15:20:02      #10 BackfillManagerTask::run() /home/couchbase/jenkins/workspace/kv_engine.threadsanitizer_neo/kv_engine/engines/ep/src/dcp/backfill-manager.cc:78 (ep-engine_ep_unit_tests+0x964e40)

15:20:02      #11 GlobalTask::execute(std::basic_string_view<char, std::char_traits<char> >) /home/couchbase/jenkins/workspace/kv_engine.threadsanitizer_neo/kv_engine/executor/globaltask.cc:98 (ep-engine_ep_unit_tests+0x1a275a9)

15:20:02      #12 FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const /home/couchbase/jenkins/workspace/kv_engine.threadsanitizer_neo/kv_engine/executor/folly_executorpool.cc:163 (ep-engine_ep_unit_tests+0x1a508fa)

15:20:02      #13 void folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&) tlm/deps/folly.exploded/include/folly/Function.h:387 (ep-engine_ep_unit_tests+0x1a50765)

15:20:02      #14 folly::detail::function::FunctionTraits<void ()>::operator()() tlm/deps/folly.exploded/include/folly/Function.h:416 (ep-engine_ep_unit_tests+0x1a3608c)

15:20:02      #15 CancellableCPUExecutor::add(GlobalTask*, folly::Function<void ()>)::$_0::operator()() const /home/couchbase/jenkins/workspace/kv_engine.threadsanitizer_neo/kv_engine/executor/cancellable_cpu_executor.cc:42 (ep-engine_ep_unit_tests+0x1a2878b)

15:20:02      #16 void folly::detail::function::FunctionTraits<void ()>::callSmall<CancellableCPUExecutor::add(GlobalTask*, folly::Function<void ()>)::$_0>(folly::detail::function::Data&) tlm/deps/folly.exploded/include/folly/Function.h:387 (ep-engine_ep_unit_tests+0x1a28275)

15:20:02      #17 folly::detail::function::FunctionTraits<void ()>::operator()() folly/Function.h:416 (ep-engine_ep_unit_tests+0x1d7ccd3)

15:20:02      #18 folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) folly/executors/ThreadPoolExecutor.cpp:97 (ep-engine_ep_unit_tests+0x1d7ccd3)

15:20:02      #19 folly::CPUThreadPoolExecutor::threadRun(std::shared_ptr<folly::ThreadPoolExecutor::Thread>) folly/executors/CPUThreadPoolExecutor.cpp:265 (ep-engine_ep_unit_tests+0x1d5fe00)

15:20:02      #20 void std::__invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&>(std::__invoke_memfun_deref, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/invoke.h:73 (ep-engine_ep_unit_tests+0x1d83b74)

15:20:02      #21 std::__invoke_result<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&>::type std::__invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&>(void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/invoke.h:95 (ep-engine_ep_unit_tests+0x1d83935)

15:20:02      #22 void std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::__call<void, , 0ul, 1ul>(std::tuple<>&&, std::_Index_tuple<0ul, 1ul>) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/functional:467 (ep-engine_ep_unit_tests+0x1d83935)

15:20:02      #23 void std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::operator()<, void>() /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/functional:549 (ep-engine_ep_unit_tests+0x1d83935)

15:20:02      #24 void folly::detail::function::FunctionTraits<void ()>::callBig<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) folly/Function.h:401 (ep-engine_ep_unit_tests+0x1d83935)

15:20:02      #25 folly::detail::function::FunctionTraits<void ()>::operator()() tlm/deps/folly.exploded/include/folly/Function.h:416 (ep-engine_ep_unit_tests+0x1a3608c)

15:20:02      #26 CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}::operator()() /home/couchbase/jenkins/workspace/kv_engine.threadsanitizer_neo/kv_engine/executor/folly_executorpool.cc:49 (ep-engine_ep_unit_tests+0x1a3b952)

15:20:02      #27 void folly::detail::function::FunctionTraits<void ()>::callBig<CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) tlm/deps/folly.exploded/include/folly/Function.h:401 (ep-engine_ep_unit_tests+0x1a3b85d)

15:20:02      #28 folly::detail::function::FunctionTraits<void ()>::operator()() tlm/deps/folly.exploded/include/folly/Function.h:416 (ep-engine_ep_unit_tests+0x1a3608c)

15:20:02      #29 folly::PriorityThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}::operator()() tlm/deps/folly.exploded/include/folly/executors/thread_factory/PriorityThreadFactory.h:54 (ep-engine_ep_unit_tests+0x1a3e888)

15:20:02      #30 void folly::detail::function::FunctionTraits<void ()>::callBig<folly::PriorityThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) tlm/deps/folly.exploded/include/folly/Function.h:401 (ep-engine_ep_unit_tests+0x1a3e73d)

15:20:02      #31 folly::detail::function::FunctionTraits<void ()>::operator()() tlm/deps/folly.exploded/include/folly/Function.h:416 (ep-engine_ep_unit_tests+0x1a3608c)

15:20:02      #32 folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}::operator()() tlm/deps/folly.exploded/include/folly/executors/thread_factory/NamedThreadFactory.h:40 (ep-engine_ep_unit_tests+0x1a3e103)

15:20:02      #33 void std::__invoke_impl<void, folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}&&) /opt/gcc-10.2.0/lib/gcc/x86_64-pc-linux-gnu/10.2.0/../../../../include/c++/10.2.0/bits/invoke.h:60 (ep-engine_ep_unit_tests+0x1a3e0ad)

15:20:02      #34 std::__invoke_result<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>::type std::__invoke<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(std::__invoke_result&&, (folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}&&)...) /opt/gcc-10.2.0/lib/gcc/x86_64-pc-linux-gnu/10.2.0/../../../../include/c++/10.2.0/bits/invoke.h:95 (ep-engine_ep_unit_tests+0x1a3e01d)

15:20:02      #35 void std::thread::_Invoker<std::tuple<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) /opt/gcc-10.2.0/lib/gcc/x86_64-pc-linux-gnu/10.2.0/../../../../include/c++/10.2.0/thread:264 (ep-engine_ep_unit_tests+0x1a3dfe5)

15:20:02      #36 std::thread::_Invoker<std::tuple<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}> >::operator()() /opt/gcc-10.2.0/lib/gcc/x86_64-pc-linux-gnu/10.2.0/../../../../include/c++/10.2.0/thread:271 (ep-engine_ep_unit_tests+0x1a3dfa5)

15:20:02      #37 std::thread::_State_impl<std::thread::_Invoker<std::tuple<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}> > >::_M_run() /opt/gcc-10.2.0/lib/gcc/x86_64-pc-linux-gnu/10.2.0/../../../../include/c++/10.2.0/thread:215 (ep-engine_ep_unit_tests+0x1a3de69)

15:20:02      #38 execute_native_thread_routine /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 (libstdc++.so.6+0xd62ef)