Thread sanitiser reports data race on xattr testsuite when running on ep-engine

WARNING: ThreadSanitizer: data race (pid=27026)

Write of size 1 at 0x7d0c0001b557 by thread T3 (mutexes: write M331):

#0 memmove <null> (memcached+0x0000004456af)

#1 unsigned char* std::__copy_move<false, true, std::random_access_iterator_tag>::__copy_m<unsigned char>(unsigned char const*, unsigned char const*, unsigned char*) /usr/bin/../lib/gcc/x86_64-linux-gnu/5.4.0/../../../../include/c++/5.4.0/bits/stl_algobase.h:384 (libxattr.so.0.1.0+0x000000003231)

#2 unsigned char* std::__copy_move_a<false, unsigned char const*, unsigned char*>(unsigned char const*, unsigned char const*, unsigned char*) /usr/bin/../lib/gcc/x86_64-linux-gnu/5.4.0/../../../../include/c++/5.4.0/bits/stl_algobase.h:401 (libxattr.so.0.1.0+0x000000003231)

#3 unsigned char* std::__copy_move_a2<false, unsigned char const*, unsigned char*>(unsigned char const*, unsigned char const*, unsigned char*) /usr/bin/../lib/gcc/x86_64-linux-gnu/5.4.0/../../../../include/c++/5.4.0/bits/stl_algobase.h:438 (libxattr.so.0.1.0+0x000000003231)

#4 unsigned char* std::copy<unsigned char const*, unsigned char*>(unsigned char const*, unsigned char const*, unsigned char*) /usr/bin/../lib/gcc/x86_64-linux-gnu/5.4.0/../../../../include/c++/5.4.0/bits/stl_algobase.h:470 (libxattr.so.0.1.0+0x000000003231)

#5 cb::xattr::Blob::set(cb::sized_buffer<unsigned char const> const&, cb::sized_buffer<unsigned char const> const&) /home/couchbase/compile/vulcan/kv_engine/xattr/blob.cc:98 (libxattr.so.0.1.0+0x000000003231)

#6 do_xattr_phase(SubdocCmdContext&) /home/couchbase/compile/vulcan/kv_engine/daemon/subdocument.cc:1062 (memcached+0x000000556d6d)

#7 subdoc_operate(SubdocCmdContext&) /home/couchbase/compile/vulcan/kv_engine/daemon/subdocument.cc:1145 (memcached+0x000000556d6d)

#8 subdoc_executor(McbpConnection&, void const*, SubdocCmdTraits) /home/couchbase/compile/vulcan/kv_engine/daemon/subdocument.cc:403 (memcached+0x000000556d6d)

#9 subdoc_multi_mutation_executor(McbpConnection*, void*) /home/couchbase/compile/vulcan/kv_engine/daemon/subdocument.cc:1716 (memcached+0x00000055bb6d)

#10 execute_request_packet(McbpConnection*) /home/couchbase/compile/vulcan/kv_engine/daemon/mcbp_executors.cc:1243 (memcached+0x000000520cd7)

#11 mcbp_execute_packet(McbpConnection*) /home/couchbase/compile/vulcan/kv_engine/daemon/mcbp_executors.cc:1354 (memcached+0x000000520cd7)

#12 conn_execute(McbpConnection*) /home/couchbase/compile/vulcan/kv_engine/daemon/statemachine_mcbp.cc:317 (memcached+0x000000556142)

#13 McbpStateMachine::execute(McbpConnection&) /home/couchbase/compile/vulcan/kv_engine/daemon/statemachine_mcbp.h:43 (memcached+0x0000005119b4)

#14 McbpConnection::runStateMachinery() /home/couchbase/compile/vulcan/kv_engine/daemon/connection_mcbp.cc:886 (memcached+0x0000005119b4)

#15 McbpConnection::runEventLoop(short) /home/couchbase/compile/vulcan/kv_engine/daemon/connection_mcbp.cc:1166 (memcached+0x0000005119b4)

#16 run_event_loop /home/couchbase/compile/vulcan/kv_engine/daemon/connections.cc:142 (memcached+0x000000513b90)

#17 event_handler(int, short, void*) /home/couchbase/compile/vulcan/kv_engine/daemon/memcached.cc:1053 (memcached+0x0000004ce11f)

#18 event_process_active_single_queue.isra.26 event.c (libevent_core.so.2.1.8+0x000000019e83)

#19 CouchbaseThread::run() /home/couchbase/compile/vulcan/platform/src/cb_pthreads.cc:59 (libplatform_so.so.0.1.0+0x000000007d52)

#20 platform_thread_wrap(void*) /home/couchbase/compile/vulcan/platform/src/cb_pthreads.cc:72 (libplatform_so.so.0.1.0+0x000000006ec8)

Previous read of size 8 at 0x7d0c0001b550 by thread T17 (mutexes: write M2102):

#0 memcpy <null> (memcached+0x0000004458ff)

#1 memcpy /usr/include/x86_64-linux-gnu/bits/string3.h:53 (libsnappy.so.1+0x00000000258d)

#2 EmitLiteral /home/couchbase/jenkins/workspace/cbdeps-platform-build/deps/packages/build/snappy/snappy-prefix/src/snappy/snappy.cc:194 (libsnappy.so.1+0x00000000258d)

#3 snappy::internal::CompressFragment(char const*, unsigned long, char*, unsigned short*, int) /home/couchbase/jenkins/workspace/cbdeps-platform-build/deps/packages/build/snappy/snappy-prefix/src/snappy/snappy.cc:449 (libsnappy.so.1+0x00000000258d)

#4 write_doc(_db*, _doc const*, unsigned long*, unsigned long*, unsigned long) /home/couchbase/compile/vulcan/couchstore/src/couch_save.cc:65 (libcouchstore.so+0x00000001a345)

#5 add_doc_to_update_list(_db*, _doc const*, _docinfo const*, _fat_buffer*, _sized_buf*, _sized_buf*, _sized_buf*, _sized_buf*, unsigned long, unsigned long) /home/couchbase/compile/vulcan/couchstore/src/couch_save.cc:304 (libcouchstore.so+0x00000001a345)

#6 couchstore_save_documents /home/couchbase/compile/vulcan/couchstore/src/couch_save.cc:385 (libcouchstore.so+0x00000001a345)

#7 CouchKVStore::saveDocs(unsigned short, unsigned long, std::vector<_doc*, std::allocator<_doc*> > const&, std::vector<_docinfo*, std::allocator<_docinfo*> >&, KVStatsCtx&, Item const*) /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/couch-kvstore/couch-kvstore.cc:1954 (ep.so+0x000000199e9e)

#8 CouchKVStore::commit2couchstore(Item const*) /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/couch-kvstore/couch-kvstore.cc:1857 (ep.so+0x000000194d03)

#9 CouchKVStore::commit(Item const*) /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/couch-kvstore/couch-kvstore.cc:1178 (ep.so+0x0000001948b1)

#10 KVBucket::commit(KVStore&, Item const*) /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/kv_bucket.cc:2050 (ep.so+0x000000106957)

#11 KVBucket::flushVBucket(unsigned short) /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/kv_bucket.cc:1991 (ep.so+0x00000010601e)

#12 Flusher::flushVB() /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/flusher.cc:281 (ep.so+0x0000000df91a)

#13 Flusher::step(GlobalTask*) /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/flusher.cc:190 (ep.so+0x0000000de869)

#14 FlusherTask::run() /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/tasks.cc:36 (ep.so+0x00000012ded8)

#15 ExecutorThread::run() /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/executorthread.cc:120 (ep.so+0x0000000d97bd)

#16 launch_executor_thread(void*) /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/executorthread.cc:33 (ep.so+0x0000000d9135)

#17 CouchbaseThread::run() /home/couchbase/compile/vulcan/platform/src/cb_pthreads.cc:59 (libplatform_so.so.0.1.0+0x000000007d52)

#18 platform_thread_wrap(void*) /home/couchbase/compile/vulcan/platform/src/cb_pthreads.cc:72 (libplatform_so.so.0.1.0+0x000000006ec8)

Location is heap block of size 45 at 0x7d0c0001b540 allocated by thread T5:

#0 operator new(unsigned long) <null> (memcached+0x0000004c9223)

#1 Blob::New(unsigned long) /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/blob.cc:32 (ep.so+0x000000027739)

#2 Item::setData(char const*, unsigned long) /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/item.h:414 (ep.so+0x0000000eb175)

#3 Item /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/item.cc:83 (ep.so+0x0000000e93d7)

#4 EventuallyPersistentEngine::itemAllocate(void**, DocKey const&, unsigned long, unsigned long, int, unsigned int, unsigned char, unsigned short) /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/ep_engine.cc:1985 (ep.so+0x0000000a01a1)

#5 EvpItemAllocate(engine_interface*, void const*, DocKey const&, unsigned long, int, unsigned int, unsigned char, unsigned short) /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/ep_engine.cc:170 (ep.so+0x00000009a6f9)

#6 EWB_Engine::allocate(engine_interface*, void const*, DocKey const&, unsigned long, int, unsigned int, unsigned char, unsigned short) /home/couchbase/compile/vulcan/kv_engine/engines/ewouldblock_engine/ewouldblock_engine.cc:330 (ewouldblock_engine.so+0x000000009314)

#7 bucket_allocate(McbpConnection*, DocKey const&, unsigned long, int, unsigned int, unsigned char, unsigned short) /home/couchbase/compile/vulcan/kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:263 (memcached+0x0000004df19f)

#8 subdoc_update(SubdocCmdContext&, ENGINE_ERROR_CODE, char const*, unsigned long, unsigned short, unsigned int) /home/couchbase/compile/vulcan/kv_engine/daemon/subdocument.cc:1201 (memcached+0x000000557394)

#9 subdoc_executor(McbpConnection&, void const*, SubdocCmdTraits) /home/couchbase/compile/vulcan/kv_engine/daemon/subdocument.cc:408 (memcached+0x000000557394)

#10 subdoc_dict_upsert_executor(McbpConnection*, void*) /home/couchbase/compile/vulcan/kv_engine/daemon/subdocument.cc:1666 (memcached+0x00000055b8ed)

#11 execute_request_packet(McbpConnection*) /home/couchbase/compile/vulcan/kv_engine/daemon/mcbp_executors.cc:1243 (memcached+0x000000520cd7)

#12 mcbp_execute_packet(McbpConnection*) /home/couchbase/compile/vulcan/kv_engine/daemon/mcbp_executors.cc:1354 (memcached+0x000000520cd7)

#13 conn_execute(McbpConnection*) /home/couchbase/compile/vulcan/kv_engine/daemon/statemachine_mcbp.cc:317 (memcached+0x000000556142)

#14 McbpStateMachine::execute(McbpConnection&) /home/couchbase/compile/vulcan/kv_engine/daemon/statemachine_mcbp.h:43 (memcached+0x0000005119b4)

#15 McbpConnection::runStateMachinery() /home/couchbase/compile/vulcan/kv_engine/daemon/connection_mcbp.cc:886 (memcached+0x0000005119b4)

#16 McbpConnection::runEventLoop(short) /home/couchbase/compile/vulcan/kv_engine/daemon/connection_mcbp.cc:1166 (memcached+0x0000005119b4)

#17 run_event_loop /home/couchbase/compile/vulcan/kv_engine/daemon/connections.cc:142 (memcached+0x000000513b90)

#18 event_handler(int, short, void*) /home/couchbase/compile/vulcan/kv_engine/daemon/memcached.cc:1053 (memcached+0x0000004ce11f)

#19 event_process_active_single_queue.isra.26 event.c (libevent_core.so.2.1.8+0x000000019e83)

#20 CouchbaseThread::run() /home/couchbase/compile/vulcan/platform/src/cb_pthreads.cc:59 (libplatform_so.so.0.1.0+0x000000007d52)

#21 platform_thread_wrap(void*) /home/couchbase/compile/vulcan/platform/src/cb_pthreads.cc:72 (libplatform_so.so.0.1.0+0x000000006ec8)

Mutex M331 (0x7d600000f998) created at:

#0 pthread_mutex_init <null> (memcached+0x00000043ce2a)

#1 cb_mutex_initialize /home/couchbase/compile/vulcan/platform/src/cb_pthreads.cc:184 (libplatform_so.so.0.1.0+0x0000000070f9)

#2 setup_thread(LIBEVENT_THREAD*) /home/couchbase/compile/vulcan/kv_engine/daemon/thread.cc:205 (memcached+0x0000004f6551)

[... cut ...]

Mutex M2102 (0x7db400000000) created at:

#0 pthread_mutex_trylock <null> (memcached+0x00000043d127)

#1 __gthread_mutex_trylock(pthread_mutex_t*) /usr/bin/../lib/gcc/x86_64-linux-gnu/5.4.0/../../../../include/x86_64-linux-gnu/c++/5.4.0/bits/gthr-default.h:757 (ep.so+0x000000105417)

#2 std::mutex::try_lock() /usr/bin/../lib/gcc/x86_64-linux-gnu/5.4.0/../../../../include/c++/5.4.0/mutex:146 (ep.so+0x000000105417)

#3 unique_lock /usr/bin/../lib/gcc/x86_64-linux-gnu/5.4.0/../../../../include/c++/5.4.0/mutex:424 (ep.so+0x000000105417)

#4 KVBucket::getLockedVBucket(unsigned short, std::try_to_lock_t) /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/kv_bucket.h:355 (ep.so+0x000000105417)

#5 KVBucket::flushVBucket(unsigned short) /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/kv_bucket.cc:1844 (ep.so+0x000000105417)

#6 Flusher::flushVB() /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/flusher.cc:281 (ep.so+0x0000000df91a)

#7 Flusher::step(GlobalTask*) /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/flusher.cc:190 (ep.so+0x0000000de869)

#8 FlusherTask::run() /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/tasks.cc:36 (ep.so+0x00000012ded8)

#9 ExecutorThread::run() /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/executorthread.cc:120 (ep.so+0x0000000d97bd)

#10 launch_executor_thread(void*) /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/executorthread.cc:33 (ep.so+0x0000000d9135)

#11 CouchbaseThread::run() /home/couchbase/compile/vulcan/platform/src/cb_pthreads.cc:59 (libplatform_so.so.0.1.0+0x000000007d52)

#12 platform_thread_wrap(void*) /home/couchbase/compile/vulcan/platform/src/cb_pthreads.cc:72 (libplatform_so.so.0.1.0+0x000000006ec8)

Thread T3 'mc:worker_1' (tid=27031, running) created by main thread at:

#0 pthread_create <null> (memcached+0x00000043c93b)

#1 cb_create_named_thread /home/couchbase/compile/vulcan/platform/src/cb_pthreads.cc:110 (libplatform_so.so.0.1.0+0x000000006e32)

#2 create_worker(void (*)(void*), void*, unsigned long*, char const*) /home/couchbase/compile/vulcan/kv_engine/daemon/thread.cc:100 (memcached+0x0000004f66de)

#3 thread_init(int, event_base*, void (*)(int, short, void*)) /home/couchbase/compile/vulcan/kv_engine/daemon/thread.cc:551 (memcached+0x0000004f66de)

[ ... cut ... ]

Thread T17 'mc:writer_0' (tid=27045, running) created by thread T11 at:

#0 pthread_create <null> (memcached+0x00000043c93b)

#1 cb_create_named_thread /home/couchbase/compile/vulcan/platform/src/cb_pthreads.cc:110 (libplatform_so.so.0.1.0+0x000000006e32)

#2 ExecutorThread::start() /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/executorthread.cc:46 (ep.so+0x0000000d8e9f)

#3 ExecutorPool::_adjustWorkers(task_type_t, unsigned long) /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/executorpool.cc:545 (ep.so+0x0000000d2655)

#4 ExecutorPool::_startWorkers() /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/executorpool.cc:616 (ep.so+0x0000000d2ff9)

#5 ExecutorPool::_registerTaskable(Taskable&) /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/executorpool.cc:499 (ep.so+0x0000000d1f16)

#6 ExecutorPool::registerTaskable(Taskable&) /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/executorpool.cc:504 (ep.so+0x0000000d1fae)

#7 KVBucket /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/kv_bucket.cc:248 (ep.so+0x0000000f631d)

#8 EPBucket /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/ep_bucket.cc:167 (ep.so+0x000000085355)

#9 __unique_if<EPBucket>::__unique_single std::make_unique<EPBucket, EventuallyPersistentEngine&>(EventuallyPersistentEngine&) /home/couchbase/compile/vulcan/platform/include/platform/make_unique.h:45 (ep.so+0x00000009fd8f)

#10 EventuallyPersistentEngine::makeBucket(Configuration&) /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/ep_engine.cc:5281 (ep.so+0x00000009fd8f)

#11 EventuallyPersistentEngine::initialize(char const*) /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/ep_engine.cc:1920 (ep.so+0x00000009fa53)

#12 EvpInitialize(engine_interface*, char const*) /home/couchbase/compile/vulcan/kv_engine/engines/ep/src/ep_engine.cc:146 (ep.so+0x00000009a557)

#13 EWB_Engine::initialize(engine_interface*, char const*) /home/couchbase/compile/vulcan/kv_engine/engines/ewouldblock_engine/ewouldblock_engine.cc:288 (ewouldblock_engine.so+0x000000008f86)

#14 CreateBucketThread::create() /home/couchbase/compile/vulcan/kv_engine/daemon/memcached.cc:2087 (memcached+0x0000004ceadb)

#15 CreateBucketThread::run() /home/couchbase/compile/vulcan/kv_engine/daemon/memcached.cc:2146 (memcached+0x0000004cf5f1)

#16 Couchbase::Thread::thread_entry() /home/couchbase/compile/vulcan/platform/src/thread.cc:45 (libplatform_so.so.0.1.0+0x00000000fd32)

#17 Couchbase::StartThreadDelegator::run(Couchbase::Thread&) /home/couchbase/compile/vulcan/platform/src/thread.cc:59 (libplatform_so.so.0.1.0+0x00000000ffb5)

#18 task_thread_main(void*) /home/couchbase/compile/vulcan/platform/src/thread.cc:65 (libplatform_so.so.0.1.0+0x00000000ffb5)

#19 CouchbaseThread::run() /home/couchbase/compile/vulcan/platform/src/cb_pthreads.cc:59 (libplatform_so.so.0.1.0+0x000000007d52)

#20 platform_thread_wrap(void*) /home/couchbase/compile/vulcan/platform/src/cb_pthreads.cc:72 (libplatform_so.so.0.1.0+0x000000006ec8)

Thread T5 'mc:worker_3' (tid=27033, running) created by main thread at:

#0 pthread_create <null> (memcached+0x00000043c93b)

#1 cb_create_named_thread /home/couchbase/compile/vulcan/platform/src/cb_pthreads.cc:110 (libplatform_so.so.0.1.0+0x000000006e32)

#2 create_worker(void (*)(void*), void*, unsigned long*, char const*) /home/couchbase/compile/vulcan/kv_engine/daemon/thread.cc:100 (memcached+0x0000004f66de)

#3 thread_init(int, event_base*, void (*)(int, short, void*)) /home/couchbase/compile/vulcan/kv_engine/daemon/thread.cc:551 (memcached+0x0000004f66de)

[ ... cut ... ]