Ephemeral Bucket :Data race when setting the rev seq no. when item is evicted

WARNING: ThreadSanitizer: data race (pid=13695)

  Write of size 8 at 0x7d1800315e58 by thread T11 (mutexes: write M3502):

    #0 StoredValue::setRevSeqno(unsigned long) /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/src/stored-value.h:496:18 (ep.so+0x0000000eda7a)

    #1 EphemeralVBucket::pageOut(HashTable::HashBucketLock const&, StoredValue*&) /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/src/ephemeral_vb.cc:107 (ep.so+0x0000000eda7a)

    #2 PagingVisitor::doEviction(HashTable::HashBucketLock const&, StoredValue*) /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/src/item_pager.cc:251:28 (ep.so+0x0000001142d3)

    #3 PagingVisitor::visit(HashTable::HashBucketLock const&, StoredValue&) /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/src/item_pager.cc:115:13 (ep.so+0x000000114133)

    #4 non-virtual thunk to PagingVisitor::visit(HashTable::HashBucketLock const&, StoredValue&) /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/src/item_pager.cc:90:10 (ep.so+0x0000001141e9)

    #5 HashTable::visit(HashTableVisitor&) /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/src/hash_table.cc:552:25 (ep.so+0x00000010bc57)

    #6 PagingVisitor::visitBucket(std::shared_ptr<VBucket>&) /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/src/item_pager.cc:139:24 (ep.so+0x000000113bb0)

    #7 VBCBAdaptor::run() /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/src/kv_bucket.cc:2130:22 (ep.so+0x000000133c54)

    #8 ExecutorThread::run() /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/src/executorthread.cc:122:39 (ep.so+0x0000000ff190)

    #9 launch_executor_thread(void*) /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/src/executorthread.cc:35:19 (ep.so+0x0000000feb05)

    #10 CouchbaseThread::run() /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/platform/src/cb_pthreads.cc:59:9 (libplatform_so.so.0.1.0+0x000000009532)

    #11 platform_thread_wrap(void*) /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/platform/src/cb_pthreads.cc:72:14 (libplatform_so.so.0.1.0+0x0000000086a8)

  Previous read of size 8 at 0x7d1800315e58 by thread T9 (mutexes: write M243911, read M2917):

    #0 StoredValue::getRevSeqno() const /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/src/stored-value.h:489:16 (ep.so+0x000000151b6a)

    #1 StoredValue::toItem(bool, unsigned short) const /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/src/stored-value.cc:199 (ep.so+0x000000151b6a)

    #2 DCPBackfillMemoryBuffered::scan() /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/src/dcp/backfill_memory.cc:249:32 (ep.so+0x00000005f49c)

    #3 DCPBackfillMemoryBuffered::run() /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/src/dcp/backfill_memory.cc:137:16 (ep.so+0x00000005ec37)

    #4 BackfillManager::backfill() /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/src/dcp/backfill-manager.cc:309:42 (ep.so+0x0000000565b7)

    #5 BackfillManagerTask::run() /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/src/dcp/backfill-manager.cc:73:41 (ep.so+0x000000055ebf)

    #6 ExecutorThread::run() /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/src/executorthread.cc:122:39 (ep.so+0x0000000ff190)

    #7 launch_executor_thread(void*) /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/src/executorthread.cc:35:19 (ep.so+0x0000000feb05)

    #8 CouchbaseThread::run() /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/platform/src/cb_pthreads.cc:59:9 (libplatform_so.so.0.1.0+0x000000009532)

    #9 platform_thread_wrap(void*) /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/platform/src/cb_pthreads.cc:72:14 (libplatform_so.so.0.1.0+0x0000000086a8)

  Location is heap block of size 82 at 0x7d1800315e40 allocated by main thread:

    #0 operator new(unsigned long) <null> (engine_testapp+0x0000004bcd33)

    #1 OrderedStoredValueFactory::operator()(Item const&, UniqueTaggedPtr<StoredValue, StoredValue::Deleter>) /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/src/stored_value_factories.h:111:22 (ep.so+0x0000000f2542)

    #2 HashTable::unlocked_addNewStoredValue(HashTable::HashBucketLock const&, Item const&) /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/src/hash_table.cc:314:14 (ep.so+0x000000109336)

    #3 EphemeralVBucket::addNewStoredValue(HashTable::HashBucketLock const&, Item const&, VBQueueItemCtx const&, GenerateRevSeqno) /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/src/ephemeral_vb.cc:428:25 (ep.so+0x0000000efb1e)

    #4 VBucket::processSet(HashTable::HashBucketLock const&, StoredValue*&, Item&, unsigned long, bool, bool, VBQueueItemCtx const&, cb::StoreIfStatus, bool, bool) /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/src/vbucket.cc:2259:34 (ep.so+0x0000001663b0)

    #5 VBucket::set(Item&, void const*, EventuallyPersistentEngine&, int, std::function<cb::StoreIfStatus (boost::optional<item_info> const&, cb::vbucket_info)>) /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/src/vbucket.cc:828:35 (ep.so+0x000000165b39)

    #6 KVBucket::set(Item&, void const*, std::function<cb::StoreIfStatus (boost::optional<item_info> const&, cb::vbucket_info)>) /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/src/kv_bucket.cc:579:20 (ep.so+0x000000123c08)

    #7 EventuallyPersistentEngine::store_if(void const*, Item&, unsigned long, ENGINE_STORE_OPERATION, std::function<cb::StoreIfStatus (boost::optional<item_info> const&, cb::vbucket_info)>) /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/src/ep_engine.cc:2240:28 (ep.so+0x0000000c5cdd)

    #8 EventuallyPersistentEngine::store(void const*, void*, unsigned long&, ENGINE_STORE_OPERATION) /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/src/ep_engine.cc:2291:15 (ep.so+0x0000000bf3c0)

    #9 EvpStore(gsl::not_null<engine_interface*>, gsl::not_null<void const*>, gsl::not_null<void*>, unsigned long&, ENGINE_STORE_OPERATION, DocumentState) /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/src/ep_engine.cc:328 (ep.so+0x0000000bf3c0)

    #10 ENGINE_ERROR_CODE std::_Bind<ENGINE_ERROR_CODE (*(engine_interface*, gsl::not_null<void const*>, gsl::not_null<void*>, std::reference_wrapper<unsigned long>, ENGINE_STORE_OPERATION, DocumentState))(gsl::not_null<engine_interface*>, gsl::not_null<void const*>, gsl::not_null<void*>, unsigned long&, ENGINE_STORE_OPERATION, DocumentState)>::__call<ENGINE_ERROR_CODE, , 0ul, 1ul, 2ul, 3ul, 4ul, 5ul>(std::tuple<>&&, std::_Index_tuple<0ul, 1ul, 2ul, 3ul, 4ul, 5ul>) /usr/bin/../lib/gcc/x86_64-linux-gnu/5.4.0/../../../../include/c++/5.4.0/functional:1073:11 (engine_testapp+0x0000004c87ca)

    #11 ENGINE_ERROR_CODE std::_Bind<ENGINE_ERROR_CODE (*(engine_interface*, gsl::not_null<void const*>, gsl::not_null<void*>, std::reference_wrapper<unsigned long>, ENGINE_STORE_OPERATION, DocumentState))(gsl::not_null<engine_interface*>, gsl::not_null<void const*>, gsl::not_null<void*>, unsigned long&, ENGINE_STORE_OPERATION, DocumentState)>::operator()<, ENGINE_ERROR_CODE>() /usr/bin/../lib/gcc/x86_64-linux-gnu/5.4.0/../../../../include/c++/5.4.0/functional:1131 (engine_testapp+0x0000004c87ca)

    #12 std::_Function_handler<ENGINE_ERROR_CODE (), std::_Bind<ENGINE_ERROR_CODE (*(engine_interface*, gsl::not_null<void const*>, gsl::not_null<void*>, std::reference_wrapper<unsigned long>, ENGINE_STORE_OPERATION, DocumentState))(gsl::not_null<engine_interface*>, gsl::not_null<void const*>, gsl::not_null<void*>, unsigned long&, ENGINE_STORE_OPERATION, DocumentState)> >::_M_invoke(std::_Any_data const&) /usr/bin/../lib/gcc/x86_64-linux-gnu/5.4.0/../../../../include/c++/5.4.0/functional:1856 (engine_testapp+0x0000004c87ca)

    #13 std::function<ENGINE_ERROR_CODE ()>::operator()() const /usr/bin/../lib/gcc/x86_64-linux-gnu/5.4.0/../../../../include/c++/5.4.0/functional:2267:14 (engine_testapp+0x0000004c4dd7)

    #14 call_engine_and_handle_EWOULDBLOCK(gsl::not_null<engine_interface*>, mock_connstruct*, std::function<ENGINE_ERROR_CODE ()>) /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/programs/engine_testapp/engine_testapp.cc:143 (engine_testapp+0x0000004c4dd7)

    #15 mock_store(gsl::not_null<engine_interface*>, gsl::not_null<void const*>, gsl::not_null<void*>, unsigned long&, ENGINE_STORE_OPERATION, DocumentState) /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/programs/engine_testapp/engine_testapp.cc:388 (engine_testapp+0x0000004c4dd7)

    #16 storeCasVb11(engine_interface*, engine_interface_v1*, void const*, ENGINE_STORE_OPERATION, char const*, char const*, unsigned long, unsigned int, unsigned long, unsigned short, unsigned int, unsigned char, DocumentState) /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/tests/ep_test_apis.cc:1044:21 (ep_testsuite_dcp.so+0x00000003ed3b)

    #17 store(engine_interface*, engine_interface_v1*, void const*, ENGINE_STORE_OPERATION, char const*, char const*, void**, unsigned long, unsigned short, unsigned int, unsigned char, DocumentState) /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/tests/ep_test_apis.cc:981:16 (ep_testsuite_dcp.so+0x000000041c66)

    #18 write_items_upto_mem_perc(engine_interface*, engine_interface_v1*, int, int, char const*, char const*) /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/tests/ep_test_apis.cc:1689:17 (ep_testsuite_dcp.so+0x000000048162)

    #19 test_dcp_cursor_dropping(engine_interface*, engine_interface_v1*) /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/tests/ep_testsuite_dcp.cc:2666:18 (ep_testsuite_dcp.so+0x00000001ca35)

    #20 execute_test(test, char const*, char const*) /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/programs/engine_testapp/engine_testapp.cc:1090:19 (engine_testapp+0x0000004c212a)

    #21 main /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/programs/engine_testapp/engine_testapp.cc:1490 (engine_testapp+0x0000004c212a)

    #22 prepare_skip_broken_under_rocks(test*) /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/tests/ep_testsuite_common.cc:265:12 (ep_testsuite_dcp.so+0x00000003ac80)

    #23 execute_test(test, char const*, char const*) /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/programs/engine_testapp/engine_testapp.cc:1063:20 (engine_testapp+0x0000004c1e99)

    #24 main /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/programs/engine_testapp/engine_testapp.cc:1490 (engine_testapp+0x0000004c1e99)