Data race on VBucket::getHighCompletedSeqno()

ThreadSanitizer: data race/home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/engines/ep/src/vbucket.cc:296VBucket::getHighCompletedSeqno() const

  Read of size 8 at 0x7b1400000c30 by thread T9 (mutexes: write M906766655311642880):

    #0 VBucket::getHighCompletedSeqno() const /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/engines/ep/src/vbucket.cc:296 (ep.so+0x0000002874af)

    #1 VBucket::_addStats(VBucketStatsDetailLevel, std::function)> const&, void const*) /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/engines/ep/src/vbucket.cc:3098 (ep.so+0x0000002874af)

    #2 EPVBucket::addStats(VBucketStatsDetailLevel, std::function)> const&, void const*) /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/engines/ep/src/ep_vb.cc:422 (ep.so+0x000000167397)

    #3 addVBStats /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/engines/ep/src/ep_engine.cc:3305 (ep.so+0x00000017926c)

    #4 EventuallyPersistentEngine::doVBucketStats(void const*, std::function)> const&, char const*, int, VBucketStatsDetailLevel) /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/engines/ep/src/ep_engine.cc:3333 (ep.so+0x00000017926c)

    #5 EventuallyPersistentEngine::getStats(void const*, cb::const_char_buffer, cb::const_char_buffer, std::function)> const&) /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/engines/ep/src/ep_engine.cc:4610 (ep.so+0x0000001916b4)

    #6 EventuallyPersistentEngine::get_stats(gsl::not_null, cb::const_char_buffer, cb::const_char_buffer, std::function)> const&) /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/engines/ep/src/ep_engine.cc:389 (ep.so+0x0000001925a6)

    #7 bucket_get_stats(Cookie&, cb::const_char_buffer, cb::sized_buffer, std::function)> const&) /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:368 (memcached+0x00000047d756)

    #8 stat_bucket_stats /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/daemon/protocol/mcbp/stats_context.cc:724 (memcached+0x00000053754e)

    #9 StatsCommandContext::doStats() /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/daemon/protocol/mcbp/stats_context.cc:855 (memcached+0x000000538158)

    #10 StatsCommandContext::step() /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/daemon/protocol/mcbp/stats_context.cc:789 (memcached+0x000000538f27)

    #11 SteppableCommandContext::drive() /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/daemon/protocol/mcbp/steppable_command_context.cc:37 (memcached+0x00000053f444)

    #12 stat_executor /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/daemon/mcbp_executors.cc:199 (memcached+0x000000505e6f)

    #13 std::_Function_handler::_M_invoke(std::_Any_data const&, Cookie&) /usr/local/include/c++/7.3.0/bits/std_function.h:316 (memcached+0x00000050e652)

    #14 std::function::operator()(Cookie&) const /usr/local/include/c++/7.3.0/bits/std_function.h:706 (memcached+0x00000050d016)

    #15 execute_client_request_packet(Cookie&, cb::mcbp::Request const&) /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/daemon/mcbp_executors.cc:855 (memcached+0x00000050d016)

    #16 execute_request_packet(Cookie&, cb::mcbp::Request const&) /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/daemon/mcbp_executors.cc:880 (memcached+0x00000050d267)

    #17 Cookie::execute() /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/daemon/cookie.cc:124 (memcached+0x0000004ebc1b)

    #18 StateMachine::conn_execute() /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/daemon/statemachine.cc:425 (memcached+0x000000545d2c)

    #19 StateMachine::execute() /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/daemon/statemachine.cc:137 (memcached+0x000000548317)

    #20 Connection::runStateMachinery() /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/daemon/connection.cc:1451 (memcached+0x0000004d02e7)

    #21 Connection::runEventLoop(short) /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/daemon/connection.cc:1525 (memcached+0x0000004d459e)

    #22 run_event_loop(Connection*, short) /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/daemon/connections.cc:148 (memcached+0x0000004e9d4b)

    #23 event_handler(int, short, void*) /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/daemon/memcached.cc:854 (memcached+0x0000004336a6)

    #24 event_persist_closure /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1580 (libevent_core.so.2.1.8+0x000000017856)

    #25 event_process_active_single_queue /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1639 (libevent_core.so.2.1.8+0x000000017856)

    #26 CouchbaseThread::run() /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/platform/src/cb_pthreads.cc:58 (libplatform_so.so.0.1.0+0x00000000a38f)

    #27 platform_thread_wrap /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/platform/src/cb_pthreads.cc:71 (libplatform_so.so.0.1.0+0x00000000a38f)

    #28   (libtsan.so.0+0x00000002843b)

  Previous write of size 8 at 0x7b1400000c30 by thread T6:

    [failed to restore the stack]

  Location is heap block of size 80 at 0x7b1400000c30 allocated by thread T6:

    #0 operator new(unsigned long)  (libtsan.so.0+0x00000006d926)

    #1 std::_MakeUniq::__single_object std::make_unique(EPStats&, PassiveDurabilityMonitor&&) /usr/local/include/c++/7.3.0/bits/unique_ptr.h:825 (ep.so+0x000000288548)

    #2 VBucket::setupSyncReplication(nlohmann::basic_json, std::allocator >, bool, long, unsigned long, double, std::allocator, nlohmann::adl_serializer> const&) /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/engines/ep/src/vbucket.cc:622 (ep.so+0x000000288548)

    #3 VBucket::setState_UNLOCKED(vbucket_state_t, nlohmann::basic_json, std::allocator >, bool, long, unsigned long, double, std::allocator, nlohmann::adl_serializer> const&, folly::SharedMutexImpl::WriteHolder const&) /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/engines/ep/src/vbucket.cc:557 (ep.so+0x00000028c8a3)

    #4 VBucketMap::setState_UNLOCKED(VBucket&, vbucket_state_t, nlohmann::basic_json, std::allocator >, bool, long, unsigned long, double, std::allocator, nlohmann::adl_serializer> const&, folly::SharedMutexImpl::WriteHolder&) /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/engines/ep/src/vbucketmap.cc:195 (ep.so+0x0000002a03d9)

    #5 KVBucket::setVBucketState_UNLOCKED(std::shared_ptr&, vbucket_state_t, nlohmann::basic_json, std::allocator >, bool, long, unsigned long, double, std::allocator, nlohmann::adl_serializer> const&, TransferVB, bool, std::unique_lock&, folly::SharedMutexImpl::WriteHolder&) /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/engines/ep/src/kv_bucket.cc:918 (ep.so+0x00000021a947)

    #6 KVBucket::setVBucketState(Vbid, vbucket_state_t, nlohmann::basic_json, std::allocator >, bool, long, unsigned long, double, std::allocator, nlohmann::adl_serializer> const&, TransferVB, void const*) /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/engines/ep/src/kv_bucket.cc:874 (ep.so+0x00000021b5a2)

    #7 PassiveStream::processSetVBucketState(SetVBucketState*) /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/engines/ep/src/dcp/passive_stream.cc:998 (ep.so+0x0000000ebcac)

    #8 PassiveStream::messageReceived(std::unique_ptr >) /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/engines/ep/src/dcp/passive_stream.cc:361 (ep.so+0x0000000f3fcf)

    #9 DcpConsumer::lookupStreamAndDispatchMessage(DcpConsumer::UpdateFlowControl&, Vbid, unsigned int, std::unique_ptr >) /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/engines/ep/src/dcp/consumer.cc:1782 (ep.so+0x0000000c0908)

    #10 DcpConsumer::setVBucketState(unsigned int, Vbid, vbucket_state_t) /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/engines/ep/src/dcp/consumer.cc:810 (ep.so+0x0000000c1141)

    #11 non-virtual thunk to EventuallyPersistentEngine::set_vbucket_state(gsl::not_null, unsigned int, Vbid, vbucket_state_t)  (ep.so+0x000000189018)

    #12 dcpSetVbucketState(Cookie&, unsigned int, Vbid, vbucket_state_t) /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:636 (memcached+0x00000047e92c)

    #13 dcp_set_vbucket_state_executor(Cookie&) /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/daemon/protocol/mcbp/dcp_set_vbucket_state_executor.cc:35 (memcached+0x00000052132b)

    #14 std::_Function_handler::_M_invoke(std::_Any_data const&, Cookie&) /usr/local/include/c++/7.3.0/bits/std_function.h:316 (memcached+0x00000050e652)

    #15 std::function::operator()(Cookie&) const /usr/local/include/c++/7.3.0/bits/std_function.h:706 (memcached+0x00000050d016)

    #16 execute_client_request_packet(Cookie&, cb::mcbp::Request const&) /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/daemon/mcbp_executors.cc:855 (memcached+0x00000050d016)

    #17 execute_request_packet(Cookie&, cb::mcbp::Request const&) /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/daemon/mcbp_executors.cc:880 (memcached+0x00000050d267)

    #18 Cookie::execute() /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/daemon/cookie.cc:124 (memcached+0x0000004ebc1b)

    #19 StateMachine::conn_execute() /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/daemon/statemachine.cc:425 (memcached+0x000000545d2c)

    #20 StateMachine::execute() /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/daemon/statemachine.cc:137 (memcached+0x000000548317)

    #21 Connection::runStateMachinery() /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/daemon/connection.cc:1451 (memcached+0x0000004d02e7)

    #22 Connection::runEventLoop(short) /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/daemon/connection.cc:1525 (memcached+0x0000004d459e)

    #23 run_event_loop(Connection*, short) /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/daemon/connections.cc:148 (memcached+0x0000004e9d4b)

    #24 event_handler(int, short, void*) /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/kv_engine/daemon/memcached.cc:854 (memcached+0x0000004336a6)

    #25 event_persist_closure /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1580 (libevent_core.so.2.1.8+0x000000017856)

    #26 event_process_active_single_queue /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1639 (libevent_core.so.2.1.8+0x000000017856)

    #27 CouchbaseThread::run() /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/platform/src/cb_pthreads.cc:58 (libplatform_so.so.0.1.0+0x00000000a38f)

    #28 platform_thread_wrap /home/couchbase/jenkins/workspace/kv_engine-madhatter-post-commit-TSan/platform/src/cb_pthreads.cc:71 (libplatform_so.so.0.1.0+0x00000000a38f)

    #29   (libtsan.so.0+0x00000002843b)