Data race during EventuallyPersistentEngine dtor with FollyExecutorPool

Note: Google Test filter = DurabilityRespondAmbiguousTest.*

[==========] Running 1 test from 1 test suite.

[----------] Global test environment set-up.

[----------] 1 test from DurabilityRespondAmbiguousTest

[ RUN      ] DurabilityRespondAmbiguousTest.RespondAmbiguousNotificationDeadLock

==================

WARNING: ThreadSanitizer: data race (pid=90420)

  Write of size 8 at 0x7b74000020a0 by main thread:

    #0 free <null> (libtsan.so.0+0x000000027806)

    #1 __gnu_cxx::new_allocator<folly::aligned<CoreLocalStats, 64ul> >::deallocate(folly::aligned<CoreLocalStats, 64ul>*, unsigned long) /usr/local/include/c++/7.3.0/ext/new_allocator.h:121 (ep-engine_ep_unit_tests+0x0000012988b1)

    #2 std::allocator_traits<std::allocator<folly::aligned<CoreLocalStats, 64ul> > >::deallocate(std::allocator<folly::aligned<CoreLocalStats, 64ul> >&, folly::aligned<CoreLocalStats, 64ul>*, unsigned long) /usr/local/include/c++/7.3.0/bits/alloc_traits.h:462 (ep-engine_ep_unit_tests+0x0000012988b1)

    #3 std::_Vector_base<folly::aligned<CoreLocalStats, 64ul>, std::allocator<folly::aligned<CoreLocalStats, 64ul> > >::_M_deallocate(folly::aligned<CoreLocalStats, 64ul>*, unsigned long) /usr/local/include/c++/7.3.0/bits/stl_vector.h:180 (ep-engine_ep_unit_tests+0x0000012988b1)

    #4 std::_Vector_base<folly::aligned<CoreLocalStats, 64ul>, std::allocator<folly::aligned<CoreLocalStats, 64ul> > >::~_Vector_base() /usr/local/include/c++/7.3.0/bits/stl_vector.h:162 (ep-engine_ep_unit_tests+0x0000012988b1)

    #5 std::vector<folly::aligned<CoreLocalStats, 64ul>, std::allocator<folly::aligned<CoreLocalStats, 64ul> > >::~vector() /usr/local/include/c++/7.3.0/bits/stl_vector.h:435 (ep-engine_ep_unit_tests+0x0000012988b1)

    #6 CoreStore<folly::aligned<CoreLocalStats, 64ul>, &cb::get_cpu_count, &cb::stripe_for_current_cpu>::~CoreStore() platform/include/platform/corestore.h:50 (ep-engine_ep_unit_tests+0x0000012988b1)

    #7 ~EPStats kv_engine/engines/ep/src/stats.cc:132 (ep-engine_ep_unit_tests+0x0000012988b1)

    #8 ~EventuallyPersistentEngine kv_engine/engines/ep/src/ep_engine.cc:6593 (ep-engine_ep_unit_tests+0x0000011e3bb5)

    #9 ~SynchronousEPEngine kv_engine/engines/ep/tests/mock/mock_synchronous_ep_engine.h:43 (ep-engine_ep_unit_tests+0x0000013e8cc5)

    #10 SynchronousEPEngineDeleter::operator()(SynchronousEPEngine*) kv_engine/engines/ep/tests/mock/mock_synchronous_ep_engine.cc:135 (ep-engine_ep_unit_tests+0x0000013e890d)

    #11 std::unique_ptr<SynchronousEPEngine, SynchronousEPEngineDeleter>::reset(SynchronousEPEngine*) /usr/local/include/c++/7.3.0/bits/unique_ptr.h:376 (ep-engine_ep_unit_tests+0x000000aecbcf)

    #12 DurabilityRespondAmbiguousTest_RespondAmbiguousNotificationDeadLock_Test::TestBody() kv_engine/engines/ep/tests/module_tests/evp_store_durability_test.cc:2350 (ep-engine_ep_unit_tests+0x000000bd3642)

...

  Previous atomic write of size 8 at 0x7b74000020a0 by thread T8:

    #0 __tsan_atomic64_fetch_sub <null> (libtsan.so.0+0x000000060890)

    #1 std::__atomic_base<long>::fetch_sub(long, std::memory_order) /usr/local/include/c++/7.3.0/bits/atomic_base.h:524 (ep-engine_ep_unit_tests+0x00000122d003)

    #2 cb::RelaxedAtomic<long>::fetch_sub(long) platform/include/relaxed_atomic.h:61 (ep-engine_ep_unit_tests+0x00000122d003)

    #3 HashTable::clear_UNLOCKED(bool) kv_engine/engines/ep/src/hash_table.cc:200 (ep-engine_ep_unit_tests+0x00000122d003)

    #4 ~HashTable kv_engine/engines/ep/src/hash_table.cc:161 (ep-engine_ep_unit_tests+0x00000122cae1)

    #5 ~VBucket kv_engine/engines/ep/src/vbucket.cc:286 (ep-engine_ep_unit_tests+0x0000012b3af4)

    #6 ~EPVBucket kv_engine/engines/ep/src/ep_vb.cc:101 (ep-engine_ep_unit_tests+0x0000011af5e1)

    #7 ~EPVBucket kv_engine/engines/ep/src/ep_vb.cc:96 (ep-engine_ep_unit_tests+0x0000011af625)

    #8 std::default_delete<VBucket>::operator()(VBucket*) const /usr/local/include/c++/7.3.0/bits/unique_ptr.h:78 (ep-engine_ep_unit_tests+0x0000012e4530)

    #9 std::unique_ptr<VBucket, std::default_delete<VBucket> >::reset(VBucket*) /usr/local/include/c++/7.3.0/bits/unique_ptr.h:376 (ep-engine_ep_unit_tests+0x0000012e4530)

    #10 ~VBucketMemoryDeletionTask kv_engine/engines/ep/src/vbucketdeletiontask.cc:45 (ep-engine_ep_unit_tests+0x0000012e4530)

    #11 void __gnu_cxx::new_allocator<VBucketMemoryAndDiskDeletionTask>::destroy<VBucketMemoryAndDiskDeletionTask>(VBucketMemoryAndDiskDeletionTask*) /usr/local/include/c++/7.3.0/ext/new_allocator.h:140 (ep-engine_ep_unit_tests+0x0000011b6c35)

    #12 void std::allocator_traits<std::allocator<VBucketMemoryAndDiskDeletionTask> >::destroy<VBucketMemoryAndDiskDeletionTask>(std::allocator<VBucketMemoryAndDiskDeletionTask>&, VBucketMemoryAndDiskDeletionTask*) /usr/local/include/c++/7.3.0/bits/alloc_traits.h:487 (ep-engine_ep_unit_tests+0x0000011b6c35)

    #13 std::_Sp_counted_ptr_inplace<VBucketMemoryAndDiskDeletionTask, std::allocator<VBucketMemoryAndDiskDeletionTask>, (__gnu_cxx::_Lock_policy)2>::_M_dispose() /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:535 (ep-engine_ep_unit_tests+0x0000011b6c35)

    #14 std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release() /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:154 (ep-engine_ep_unit_tests+0x000001221e75)

    #15 std::__shared_count<(__gnu_cxx::_Lock_policy)2>::~__shared_count() /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:684 (ep-engine_ep_unit_tests+0x000001221e75)

    #16 std::__shared_ptr<GlobalTask, (__gnu_cxx::_Lock_policy)2>::~__shared_ptr() /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:1123 (ep-engine_ep_unit_tests+0x000001221e75)

    #17 std::__shared_ptr<GlobalTask, (__gnu_cxx::_Lock_policy)2>::reset() /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:1235 (ep-engine_ep_unit_tests+0x000001221e75)

    #18 FollyExecutorPool::TaskProxy::~TaskProxy()::{lambda()#1}::operator()() kv_engine/engines/ep/src/folly_executorpool.cc:80 (ep-engine_ep_unit_tests+0x000001221e75)

    #19 void folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::~TaskProxy()::{lambda()#1}>(folly::detail::function::Data&) build/tlm/deps/folly.exploded/include/folly/Function.h:387 (ep-engine_ep_unit_tests+0x000001221e75)

    #20 folly::detail::function::FunctionTraits<void ()>::operator()() follytsan/folly/Function.h:416 (ep-engine_ep_unit_tests+0x000001503974)

    #21 folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) follytsan/folly/executors/ThreadPoolExecutor.cpp:98 (ep-engine_ep_unit_tests+0x000001503974)

    #22 folly::CPUThreadPoolExecutor::threadRun(std::shared_ptr<folly::ThreadPoolExecutor::Thread>) follytsan/folly/executors/CPUThreadPoolExecutor.cpp:265 (ep-engine_ep_unit_tests+0x0000014db538)

...

SUMMARY: ThreadSanitizer: data race (install/lib/libtsan.so.0+0x27806) in free