TSan: data race on Cookie::setEngineStorage when called by background thread

WARNING: ThreadSanitizer: data race (pid=46769)

  Write of size 8 at 0x7b5400291698 by thread T60 (mutexes: read M542537949550136824, write M360774, read M541974810616805208, write M1133353977207195360):

    #0 Cookie::setEngineStorage(void*) /home/daver/repos/couchbase/server/kv_engine/daemon/cookie.h:432 (memcached+0x66a6e1)

    #1 EventuallyPersistentEngine::storeEngineSpecific(CookieIface const*, void*) /home/daver/repos/couchbase/server/kv_engine/engines/ep/src/ep_engine.cc:1841 (memcached+0x7c4162)

    #2 operator() /home/daver/repos/couchbase/server/kv_engine/engines/ep/src/kv_bucket.cc:2756 (memcached+0xa65268)

    #3 __invoke_impl<void, KVBucket::makeSyncWriteCompleteCB()::<lambda(const CookieIface*, cb::engine_errc)>&, const CookieIface*, cb::engine_errc> /usr/include/c++/10/bits/invoke.h:60 (memcached+0xa65268)

    #4 __invoke_r<void, KVBucket::makeSyncWriteCompleteCB()::<lambda(const CookieIface*, cb::engine_errc)>&, const CookieIface*, cb::engine_errc> /usr/include/c++/10/bits/invoke.h:110 (memcached+0xa65268)

    #5 _M_invoke /usr/include/c++/10/bits/std_function.h:291 (memcached+0xa65268)

    #6 std::function<void (CookieIface const*, cb::engine_errc)>::operator()(CookieIface const*, cb::engine_errc) const /usr/include/c++/10/bits/std_function.h:622 (memcached+0x9bf920)

    #7 VBucket::notifyClientOfSyncWriteComplete(CookieIface const*, cb::engine_errc) /home/daver/repos/couchbase/server/kv_engine/engines/ep/src/vbucket.cc:1041 (memcached+0x9bf920)

    #8 VBucket::abort(DocKey const&, unsigned long, std::optional<long>, Collections::VB::CachingReadHandle const&, CookieIface const*) /home/daver/repos/couchbase/server/kv_engine/engines/ep/src/vbucket.cc:1022 (memcached+0xa16366)

    #9 ActiveDurabilityMonitor::abort(DurabilityMonitor::ActiveSyncWrite const&) /home/daver/repos/couchbase/server/kv_engine/engines/ep/src/durability/active_durability_monitor.cc:930 (memcached+0x858ec3)

    #10 ActiveDurabilityMonitor::processCompletedSyncWriteQueue() /home/daver/repos/couchbase/server/kv_engine/engines/ep/src/durability/active_durability_monitor.cc:484 (memcached+0x859a1f)

    #11 VBucket::processResolvedSyncWrites() /home/daver/repos/couchbase/server/kv_engine/engines/ep/src/vbucket.cc:710 (memcached+0x9cb52f)

    #12 DurabilityCompletionTask::visitVBucket(VBucket&) /home/daver/repos/couchbase/server/kv_engine/engines/ep/src/durability/durability_completion_task.cc:30 (memcached+0x845ab5)

    #13 VBNotifiableTask::run() /home/daver/repos/couchbase/server/kv_engine/engines/ep/src/vb_notifiable_task.cc:53 (memcached+0xa39035)

    #14 GlobalTask::execute() /home/daver/repos/couchbase/server/kv_engine/executor/globaltask.cc:68 (memcached+0xdcd5ed)

    #15 FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const /home/daver/repos/couchbase/server/kv_engine/executor/folly_executorpool.cc:189 (memcached+0xdcd5ed)

    #16 void folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&) tlm/deps/folly.exploded/include/folly/Function.h:387 (memcached+0xdce315)

    #17 folly::detail::function::FunctionTraits<void ()>::operator()() folly/Function.h:416 (memcached+0xfb15a3)

    #18 folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) folly/executors/ThreadPoolExecutor.cpp:97 (memcached+0xfb15a3)

    #19 folly::CPUThreadPoolExecutor::threadRun(std::shared_ptr<folly::ThreadPoolExecutor::Thread>) folly/executors/CPUThreadPoolExecutor.cpp:265 (memcached+0xf8ce50)

    #20 void std::__invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&>(std::__invoke_memfun_deref, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/invoke.h:73 (memcached+0xfb8774)

    #21 std::__invoke_result<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&>::type std::__invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&>(void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/invoke.h:95 (memcached+0xfb8535)

    #22 void std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::__call<void, , 0ul, 1ul>(std::tuple<>&&, std::_Index_tuple<0ul, 1ul>) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/functional:467 (memcached+0xfb8535)

    #23 void std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::operator()<, void>() /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/functional:549 (memcached+0xfb8535)

    #24 void folly::detail::function::FunctionTraits<void ()>::callBig<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) folly/Function.h:401 (memcached+0xfb8535)

    #25 folly::detail::function::FunctionTraits<void ()>::operator()() tlm/deps/folly.exploded/include/folly/Function.h:416 (memcached+0xdccd15)

    #26 CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}::operator()() /home/daver/repos/couchbase/server/kv_engine/executor/folly_executorpool.cc:47 (memcached+0xdccd15)

    #27 void folly::detail::function::FunctionTraits<void ()>::callBig<CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) tlm/deps/folly.exploded/include/folly/Function.h:401 (memcached+0xdccd15)

    #28 folly::detail::function::FunctionTraits<void ()>::operator()() tlm/deps/folly.exploded/include/folly/Function.h:416 (memcached+0xdcbd6b)

    #29 folly::PriorityThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}::operator()() tlm/deps/folly.exploded/include/folly/executors/thread_factory/PriorityThreadFactory.h:54 (memcached+0xdcbd6b)

    #30 void folly::detail::function::FunctionTraits<void ()>::callBig<folly::PriorityThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) tlm/deps/folly.exploded/include/folly/Function.h:401 (memcached+0xdcbd6b)

    #31 folly::detail::function::FunctionTraits<void ()>::operator()() tlm/deps/folly.exploded/include/folly/Function.h:416 (memcached+0xdcb6da)

    #32 folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}::operator()() tlm/deps/folly.exploded/include/folly/executors/thread_factory/NamedThreadFactory.h:40 (memcached+0xdcb6da)

    #33 void std::__invoke_impl<void, folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}&&) /usr/include/c++/10/bits/invoke.h:60 (memcached+0xdcb6da)

    #34 std::__invoke_result<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>::type std::__invoke<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}&&) /usr/include/c++/10/bits/invoke.h:95 (memcached+0xdcb6da)

    #35 void std::thread::_Invoker<std::tuple<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) /usr/include/c++/10/thread:264 (memcached+0xdcb6da)

    #36 std::thread::_Invoker<std::tuple<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}> >::operator()() /usr/include/c++/10/thread:271 (memcached+0xdcb6da)

    #37 std::thread::_State_impl<std::thread::_Invoker<std::tuple<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}> > >::_M_run() /usr/include/c++/10/thread:215 (memcached+0xdcb6da)

    #38 <null> <null> (libstdc++.so.6+0xd44bf)

  Previous write of size 8 at 0x7b5400291698 by thread T18 (mutexes: write M3809):

    #0 Cookie::setEngineStorage(void*) /home/daver/repos/couchbase/server/kv_engine/daemon/cookie.h:432 (memcached+0x66a6e1)

    #1 EventuallyPersistentEngine::storeEngineSpecific(CookieIface const*, void*) /home/daver/repos/couchbase/server/kv_engine/engines/ep/src/ep_engine.cc:1841 (memcached+0x7bb0c2)

    #2 EventuallyPersistentEngine::storeIfInner(CookieIface const*, Item&, unsigned long, StoreSemantics, std::function<cb::StoreIfStatus (std::optional<item_info> const&, cb::vbucket_info)> const&, bool) /home/daver/repos/couchbase/server/kv_engine/engines/ep/src/ep_engine.cc:2539 (memcached+0x7de96a)

    #3 EventuallyPersistentEngine::store_if(CookieIface const&, ItemIface&, unsigned long, StoreSemantics, std::function<cb::StoreIfStatus (std::optional<item_info> const&, cb::vbucket_info)> const&, std::optional<cb::durability::Requirements> const&, DocumentState, bool) /home/daver/repos/couchbase/server/kv_engine/engines/ep/src/ep_engine.cc:478 (memcached+0x7debed)

    #4 bucket_store_if(Cookie&, gsl::not_null<ItemIface*>, unsigned long, StoreSemantics, std::function<cb::StoreIfStatus (std::optional<item_info> const&, cb::vbucket_info)>, std::optional<cb::durability::Requirements>, DocumentState, bool) /home/daver/repos/couchbase/server/kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:148 (memcached+0x7517bd)

    #5 MutationCommandContext::storeItem() /home/daver/repos/couchbase/server/kv_engine/daemon/protocol/mcbp/mutation_context.cc:288 (memcached+0x733f6d)

    #6 MutationCommandContext::step() /home/daver/repos/couchbase/server/kv_engine/daemon/protocol/mcbp/mutation_context.cc:54 (memcached+0x7385f7)

    #7 SteppableCommandContext::drive() /home/daver/repos/couchbase/server/kv_engine/daemon/protocol/mcbp/steppable_command_context.cc:36 (memcached+0x71f714)

    #8 add_set_replace_executor /home/daver/repos/couchbase/server/kv_engine/daemon/mcbp_executors.cc:137 (memcached+0x5d5b3a)

    #9 set_executor /home/daver/repos/couchbase/server/kv_engine/daemon/mcbp_executors.cc:145 (memcached+0x5d5b3a)

    #10 void std::__invoke_impl<void, void (*&)(Cookie&), Cookie&>(std::__invoke_other, void (*&)(Cookie&), Cookie&) /usr/include/c++/10/bits/invoke.h:60 (memcached+0x5f8342)

    #11 std::enable_if<is_invocable_r_v<void, void (*&)(Cookie&), Cookie&>, void>::type std::__invoke_r<void, void (*&)(Cookie&), Cookie&>(void (*&)(Cookie&), Cookie&) /usr/include/c++/10/bits/invoke.h:110 (memcached+0x5f8342)

    #12 std::_Function_handler<void (Cookie&), void (*)(Cookie&)>::_M_invoke(std::_Any_data const&, Cookie&) /usr/include/c++/10/bits/std_function.h:291 (memcached+0x5f8342)

    #13 std::function<void (Cookie&)>::operator()(Cookie&) const /usr/include/c++/10/bits/std_function.h:622 (memcached+0x5eed6c)

    #14 execute_client_request_packet(Cookie&, cb::mcbp::Request const&) /home/daver/repos/couchbase/server/kv_engine/daemon/mcbp_executors.cc:856 (memcached+0x5eed6c)

    #15 execute_request_packet(Cookie&, cb::mcbp::Request const&) /home/daver/repos/couchbase/server/kv_engine/daemon/mcbp_executors.cc:864 (memcached+0x5eee51)

    #16 Cookie::doExecute() /home/daver/repos/couchbase/server/kv_engine/daemon/cookie.cc:151 (memcached+0x6666ac)

    #17 Cookie::execute(bool) /home/daver/repos/couchbase/server/kv_engine/daemon/cookie.cc:168 (memcached+0x669908)

    #18 Connection::executeCommandPipeline() /home/daver/repos/couchbase/server/kv_engine/daemon/connection.cc:569 (memcached+0x69658d)

    #19 Connection::executeCommandsCallback() /home/daver/repos/couchbase/server/kv_engine/daemon/connection.cc:793 (memcached+0x696be8)

    #20 Connection::rw_callback(bufferevent*, void*) /home/daver/repos/couchbase/server/kv_engine/daemon/connection.cc:942 (memcached+0x697851)

    #21 bufferevent_run_deferred_callbacks_unlocked /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/bufferevent.c:208 (libevent_core-2.1.so.7+0xf71d)

    #22 folly::EventBase::loopBody(int, bool) folly/io/async/EventBase.cpp:397 (memcached+0xfc9b52)

    #23 folly::EventBase::loop() folly/io/async/EventBase.cpp:315 (memcached+0xfcb06b)

    #24 folly::EventBase::loopForever() folly/io/async/EventBase.cpp:538 (memcached+0xfcb06b)

    #25 worker_libevent /home/daver/repos/couchbase/server/kv_engine/daemon/thread.cc:115 (memcached+0x6c16af)

    #26 CouchbaseThread::run() /home/daver/repos/couchbase/server/platform/src/cb_pthreads.cc:51 (memcached+0xf217d5)

    #27 platform_thread_wrap /home/daver/repos/couchbase/server/platform/src/cb_pthreads.cc:64 (memcached+0xf217d5)