Details
-
Bug
-
Resolution: Fixed
-
Critical
-
.master
-
Security Level: Public
-
Untriaged
-
Unknown
Description
As detected by ThreadSanitizer, there is a data race caused by DCP compression; between the DcpProducer thread compressing an item and the Flusher - see ThreadSanitizer log below:
I think this has the potential to loose / corrupt data.
ARNING: ThreadSanitizer: data race (pid=17884)
|
Write of size 8 at 0x7d14000dcd98 by thread T14 (mutexes: write M149761):
|
#0 SingleThreadedRCPtr<Blob>::swap() ep-engine/src/atomic.h:322:9 (ep.so+0x00000005b79d)
|
#1 SingleThreadedRCPtr<Blob>::reset() ep-engine/src/atomic.h:279 (ep.so+0x00000005b79d)
|
#2 Item::setData() ep-engine/src/item.h:677 (ep.so+0x00000005b79d)
|
#3 Item::compressValue() ep-engine/src/item.h:434:17 (ep.so+0x0000000661df)
|
#4 DcpProducer::getNextItem() ep-engine/src/dcp/producer.cc:787:50 (ep.so+0x000000060c42)
|
#5 DcpProducer::step() ep-engine/src/dcp/producer.cc:269:16 (ep.so+0x00000005fff1)
|
#6 EvpDcpStep() ep-engine/src/ep_engine.cc:1481:23 (ep.so+0x0000000ab82b)
|
#7 mock_dcp_step() memcached/programs/engine_testapp/engine_testapp.cc:476:12 (engine_testapp+0x0000004c5b25)
|
#8 perf_dcp_client() ep-engine/tests/ep_perfsuite.cc:615:33 (ep_perfsuite.so+0x00000000ff48)
|
#9 dcp_client_thread() ep-engine/tests/ep_perfsuite.cc:669 (ep_perfsuite.so+0x00000000ff48)
|
#10 platform_thread_wrap platform/src/cb_pthreads.c:23:5 (libplatform.so.0.1.0+0x000000004161)
|
|
Previous read of size 8 at 0x7d14000dcd98 by thread T7 (mutexes: write M11737):
|
#0 SingleThreadedRCPtr<Blob>::gimme() const ep-engine/src/atomic.h:317:16 (ep.so+0x0000001524d2)
|
#1 SingleThreadedRCPtr<Blob>::SingleThreadedRCPtr() ep-engine/src/atomic.h:267 (ep.so+0x0000001524d2)
|
#2 CouchRequest::CouchRequest() ep-engine/src/couch-kvstore/couch-kvstore.cc:198 (ep.so+0x0000001524d2)
|
#3 CouchKVStore::set() ep-engine/src/couch-kvstore/couch-kvstore.cc:388:25 (ep.so+0x000000157a98)
|
#4 EventuallyPersistentStore::flushOneDelOrSet() ep-engine/src/ep.cc:3411:9 (ep.so+0x0000000925bc)
|
#5 EventuallyPersistentStore::flushVBucket() ep-engine/src/ep.cc:3264:47 (ep.so+0x000000091115)
|
#6 Flusher::flushVB() ep-engine/src/flusher.cc:296:13 (ep.so+0x0000000f05f5)
|
#7 Flusher::step() ep-engine/src/flusher.cc:186:9 (ep.so+0x0000000eec85)
|
#8 FlusherTask::run() ep-engine/src/tasks.cc:63:12 (ep.so+0x0000001292f2)
|
#9 ExecutorThread::run() ep-engine/src/executorthread.cc:115:26 (ep.so+0x0000000ea40d)
|
#10 launch_executor_thread() ep-engine/src/executorthread.cc:33:9 (ep.so+0x0000000e9fd5)
|
#11 platform_thread_wrap platform/src/cb_pthreads.c:23:5 (libplatform.so.0.1.0+0x000000004161)
|
|
Location is heap block of size 72 at 0x7d14000dcd70 allocated by thread T13:
|
#0 operator new() <null> (engine_testapp+0x00000045cded)
|
#1 StoredValue::toItem() const ep-engine/src/stored-value.cc:761:17 (ep.so+0x00000010c7bf)
|
#2 EventuallyPersistentStore::queueDirty() ep-engine/src/ep.cc:3437:24 (ep.so+0x00000007dc01)
|
#3 EventuallyPersistentStore::set() ep-engine/src/ep.cc:861:9 (ep.so+0x00000007f91f)
|
#4 EventuallyPersistentEngine::store() ep-engine/src/ep_engine.cc:2265:15 (ep.so+0x0000000adfcf)
|
#5 EvpStore() ep-engine/src/ep_engine.cc:231:38 (ep.so+0x0000000aade7)
|
#6 mock_store() memcached/programs/engine_testapp/engine_testapp.cc:272:19 (engine_testapp+0x0000004c5112)
|
#7 storeCasVb11() ep-engine/tests/ep_test_apis.cc:846:10 (ep_perfsuite.so+0x000000020bef)
|
#8 store() ep-engine/tests/ep_test_apis.cc:797:12 (ep_perfsuite.so+0x00000001edcb)
|
#9 perf_load_client() ep-engine/tests/ep_perfsuite.cc:555:19 (ep_perfsuite.so+0x00000000f325)
|
#10 load_thread() ep-engine/tests/ep_perfsuite.cc:659 (ep_perfsuite.so+0x00000000f325)
|
#11 platform_thread_wrap platform/src/cb_pthreads.c:23:5 (libplatform.so.0.1.0+0x000000004161)
|
|
Mutex M149761 (0x7d4c00023b08) created at:
|
#0 pthread_mutex_init <null> (engine_testapp+0x0000004616c0)
|
#1 cb_mutex_initialize platform/src/cb_pthreads.c:105:5 (libplatform.so.0.1.0+0x000000004257)
|
#2 Mutex::Mutex() ep-engine/src/mutex.cc:23:5 (ep.so+0x00000010703e)
|
#3 Producer::Producer() ep-engine/src/tapconnection.h:785:5 (ep.so+0x000000064394)
|
#4 DcpProducer::DcpProducer() ep-engine/src/dcp/producer.cc:50:54 (ep.so+0x00000005dfac)
|
#5 DcpConnMap::newProducer() ep-engine/src/connmap.cc:1026:24 (ep.so+0x0000000441d4)
|
#6 EventuallyPersistentEngine::dcpOpen() ep-engine/src/ep_engine.cc:6201:19 (ep.so+0x0000000c7bdd)
|
#7 EvpDcpOpen() ep-engine/src/ep_engine.cc:1497:19 (ep.so+0x0000000ab8af)
|
#8 mock_dcp_open() memcached/programs/engine_testapp/engine_testapp.cc:488:12 (engine_testapp+0x0000004c5ba5)
|
#9 perf_dcp_client() ep-engine/tests/ep_perfsuite.cc:573:15 (ep_perfsuite.so+0x00000000fc1a)
|
#10 dcp_client_thread() ep-engine/tests/ep_perfsuite.cc:669 (ep_perfsuite.so+0x00000000fc1a)
|
#11 platform_thread_wrap platform/src/cb_pthreads.c:23:5 (libplatform.so.0.1.0+0x000000004161)
|
|
Mutex M11737 (0x7dc400014050) created at:
|
#0 pthread_mutex_init <null> (engine_testapp+0x0000004616c0)
|
#1 cb_mutex_initialize platform/src/cb_pthreads.c:105:5 (libplatform.so.0.1.0+0x000000004257)
|
#2 Mutex::Mutex() ep-engine/src/mutex.cc:23:5 (ep.so+0x00000010703e)
|
#3 EventuallyPersistentStore::EventuallyPersistentStore() ep-engine/src/ep.cc:302:18 (ep.so+0x000000077d07)
|
#4 EventuallyPersistentEngine::initialize() ep-engine/src/ep_engine.cc:2160:15 (ep.so+0x0000000ad083)
|
#5 EvpInitialize() ep-engine/src/ep_engine.cc:136:38 (ep.so+0x0000000aa9d8)
|
#6 init_engine_instance memcached/utilities/engine_loader.c:157:17 (libmcd_util.so.1.0.0+0x00000000580b)
|
#7 create_bucket() memcached/programs/engine_testapp/engine_testapp.cc:980:17 (engine_testapp+0x0000004c49a7)
|
#8 execute_test() memcached/programs/engine_testapp/engine_testapp.cc:1083:25 (engine_testapp+0x0000004c40b9)
|
#9 main memcached/programs/engine_testapp/engine_testapp.cc:1439 (engine_testapp+0x0000004c40b9)
|
|
SUMMARY: ThreadSanitizer: data race ep-engine/src/atomic.h:322 SingleThreadedRCPtr<Blob>::swap(Blob*)
|
Steps to reproduce
1. Build with ThreadSanitizer:
CC=clang-3.6 CXX=clang++-3.6 make EXTRA_CMAKE_OPTIONS="-D CB_THREADSANITIZER=1" -j8
2. Run:
TSAN_OPTIONS="external_symbolizer_path=/usr/bin/llvm-symbolizer-3.6 suppressions=/home/couchbase/couchbase/tlm/tsan.suppressions second_deadlock_stack=1" "/home/couchbase/couchbase/build/memcached/engine_testapp" "-E" "ep.so" "-T" "ep_perfsuite.so"