Details
-
Bug
-
Resolution: Fixed
-
Major
-
6.5.0
-
Triaged
-
Unknown
-
KV-Engine MH 2nd Beta
Description
As seen in KV-Engine TSan post-commit, there is a race on VBucket::purge_seqno:
ThreadSanitizer: data race kv_engine/engines/ep/src/monotonic.h:134Monotonic::operator=(unsigned long const&)
|
|
Write of size 8 at 0x7b6800107728 by thread T41:
|
#0 Monotonic::operator=(unsigned long const&) kv_engine/engines/ep/src/monotonic.h:134 (ep.so+0x0000001be09e)
|
#1 VBucket::setPurgeSeqno(unsigned long) kv_engine/engines/ep/src/vbucket.h:218 (ep.so+0x0000001be09e)
|
#2 EphemeralVBucket::purgeStaleItems(std::function) kv_engine/engines/ep/src/ephemeral_vb.cc:350 (ep.so+0x0000001be09e)
|
#3 EphemeralVBucket::StaleItemDeleter::visit(VBucket&) kv_engine/engines/ep/src/ephemeral_tombstone_purger.cc:205 (ep.so+0x0000001b7571)
|
#4 KVBucket::pauseResumeVisit(PauseResumeVBVisitor&, KVBucketIface::Position&) kv_engine/engines/ep/src/kv_bucket.cc:2278 (ep.so+0x000000209a09)
|
#5 EphTombstoneStaleItemDeleter::run() kv_engine/engines/ep/src/ephemeral_tombstone_purger.cc:273 (ep.so+0x0000001b6d59)
|
#6 ExecutorThread::run() kv_engine/engines/ep/src/executorthread.cc:153 (ep.so+0x0000001d0405)
|
#7 launch_executor_thread kv_engine/engines/ep/src/executorthread.cc:34 (ep.so+0x0000001d1345)
|
#8 CouchbaseThread::run() platform/src/cb_pthreads.cc:58 (libplatform_so.so.0.1.0+0x000000009c5f)
|
#9 platform_thread_wrap platform/src/cb_pthreads.cc:71 (libplatform_so.so.0.1.0+0x000000009c5f)
|
#10 (libtsan.so.0+0x000000024feb)
|
|
Previous read of size 8 at 0x7b6800107728 by thread T6 (mutexes: write M1057918717805263064):
|
#0 VBucket::_addStats(bool, std::function)> const&, void const*) kv_engine/engines/ep/src/vbucket.cc:3010 (ep.so+0x0000002796b6)
|
#1 EphemeralVBucket::addStats(bool, std::function)> const&, void const*) kv_engine/engines/ep/src/ephemeral_vb.cc:166 (ep.so+0x0000001b8824)
|
#2 addVBStats kv_engine/engines/ep/src/ep_engine.cc:3155 (ep.so+0x000000183e93)
|
#3 visitBucket kv_engine/engines/ep/src/ep_engine.cc:3127 (ep.so+0x000000183e93)
|
#4 KVBucket::visit(VBucketVisitor&) kv_engine/engines/ep/src/kv_bucket.cc:2254 (ep.so+0x000000209efd)
|
#5 EventuallyPersistentEngine::doVBucketStats(void const*, std::function)> const&, char const*, int, bool, bool) kv_engine/engines/ep/src/ep_engine.cc:3193 (ep.so+0x000000183a7b)
|
#6 EventuallyPersistentEngine::getStats(void const*, cb::const_char_buffer, cb::const_char_buffer, std::function)> const&) kv_engine/engines/ep/src/ep_engine.cc:4419 (ep.so+0x000000192c9e)
|
#7 EventuallyPersistentEngine::get_stats(gsl::not_null, cb::const_char_buffer, cb::const_char_buffer, std::function)> const&) kv_engine/engines/ep/src/ep_engine.cc:372 (ep.so+0x000000193baf)
|
#8 bucket_get_stats(Cookie&, cb::const_char_buffer, cb::sized_buffer, std::function)> const&) kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:347 (memcached+0x00000047c516)
|
#9 stat_bucket_stats kv_engine/daemon/protocol/mcbp/stats_context.cc:715 (memcached+0x000000529afe)
|
#10 StatsCommandContext::doStats() kv_engine/daemon/protocol/mcbp/stats_context.cc:846 (memcached+0x00000052a708)
|
#11 StatsCommandContext::step() kv_engine/daemon/protocol/mcbp/stats_context.cc:780 (memcached+0x00000052b4d7)
|
#12 SteppableCommandContext::drive() kv_engine/daemon/protocol/mcbp/steppable_command_context.cc:37 (memcached+0x000000531934)
|
#13 stat_executor kv_engine/daemon/mcbp_executors.cc:199 (memcached+0x0000004faf2f)
|
#14 std::_Function_handler::_M_invoke(std::_Any_data const&, Cookie&) /usr/local/include/c++/7.3.0/bits/std_function.h:316 (memcached+0x000000503592)
|
#15 std::function::operator()(Cookie&) const /usr/local/include/c++/7.3.0/bits/std_function.h:706 (memcached+0x000000501d8e)
|
#16 execute_client_request_packet(Cookie&, cb::mcbp::Request const&) kv_engine/daemon/mcbp_executors.cc:857 (memcached+0x000000501d8e)
|
#17 execute_request_packet(Cookie&, cb::mcbp::Request const&) kv_engine/daemon/mcbp_executors.cc:881 (memcached+0x000000501f67)
|
#18 Cookie::execute() kv_engine/daemon/cookie.cc:124 (memcached+0x0000004e507b)
|
#19 StateMachine::conn_execute() kv_engine/daemon/statemachine.cc:410 (memcached+0x000000537eec)
|
#20 StateMachine::execute() kv_engine/daemon/statemachine.cc:137 (memcached+0x00000053a227)
|
#21 Connection::runStateMachinery() kv_engine/daemon/connection.cc:1377 (memcached+0x0000004cb3f7)
|
#22 Connection::runEventLoop(short) kv_engine/daemon/connection.cc:1451 (memcached+0x0000004cdbfe)
|
#23 run_event_loop(Connection*, short) kv_engine/daemon/connections.cc:148 (memcached+0x0000004e349b)
|
#24 event_handler(int, short, void*) kv_engine/daemon/memcached.cc:840 (memcached+0x000000432d2d)
|
#25 event_persist_closure /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1580 (libevent_core.so.2.1.8+0x000000017086)
|
#26 event_process_active_single_queue /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1639 (libevent_core.so.2.1.8+0x000000017086)
|
#27 CouchbaseThread::run() platform/src/cb_pthreads.cc:58 (libplatform_so.so.0.1.0+0x000000009c5f)
|
#28 platform_thread_wrap platform/src/cb_pthreads.cc:71 (libplatform_so.so.0.1.0+0x000000009c5f)
|
#29 (libtsan.so.0+0x000000024feb)
|
|
Location is heap block of size 1312 at 0x7b6800107400 allocated by thread T9:
|
#0 operator new(unsigned long) (libtsan.so.0+0x00000006a4d6)
|
#1 EphemeralBucket::makeVBucket(Vbid, vbucket_state_t, KVShard*, std::unique_ptr >, std::unique_ptr, std::default_delete > >, std::unique_ptr >, vbucket_state_t, long, unsigned long, unsigned long, unsigned long, unsigned long, long, bool, nlohmann::basic_json, std::allocator >, bool, long, unsigned long, double, std::allocator, nlohmann::adl_serializer> const&) kv_engine/engines/ep/src/ephemeral_bucket.cc:221 (ep.so+0x0000001b1123)
|
#2 KVBucket::setVBucketState_UNLOCKED(Vbid, vbucket_state_t, nlohmann::basic_json, std::allocator >, bool, long, unsigned long, double, std::allocator, nlohmann::adl_serializer> const&, TransferVB, bool, std::unique_lock&, folly::SharedMutexImpl::WriteHolder*) kv_engine/engines/ep/src/kv_bucket.cc:968 (ep.so+0x000000219da6)
|
#3 KVBucket::setVBucketState(Vbid, vbucket_state_t, nlohmann::basic_json, std::allocator >, bool, long, unsigned long, double, std::allocator, nlohmann::adl_serializer> const&, TransferVB, void const*) kv_engine/engines/ep/src/kv_bucket.cc:829 (ep.so+0x00000021ae6e)
|
#4 EventuallyPersistentEngine::setVBucketState(void const*, std::function const&, Vbid, vbucket_state_t, nlohmann::basic_json, std::allocator >, bool, long, unsigned long, double, std::allocator, nlohmann::adl_serializer> const&, TransferVB, unsigned long) kv_engine/engines/ep/src/ep_engine.cc:6298 (ep.so+0x000000173132)
|
#5 setVBucket kv_engine/engines/ep/src/ep_engine.cc:904 (ep.so+0x000000190739)
|
#6 processUnknownCommand kv_engine/engines/ep/src/ep_engine.cc:1115 (ep.so+0x000000190739)
|
#7 EventuallyPersistentEngine::unknown_command(void const*, cb::mcbp::Request const&, std::function const&) kv_engine/engines/ep/src/ep_engine.cc:1207 (ep.so+0x000000190739)
|
#8 bucket_unknown_command(Cookie&, std::function const&) kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:37 (memcached+0x00000047b65e)
|
#9 process_bin_unknown_packet kv_engine/daemon/mcbp_executors.cc:126 (memcached+0x0000004f9a83)
|
#10 std::_Function_handler::_M_invoke(std::_Any_data const&, Cookie&) /usr/local/include/c++/7.3.0/bits/std_function.h:316 (memcached+0x000000503592)
|
#11 std::function::operator()(Cookie&) const /usr/local/include/c++/7.3.0/bits/std_function.h:706 (memcached+0x000000501d8e)
|
#12 execute_client_request_packet(Cookie&, cb::mcbp::Request const&) kv_engine/daemon/mcbp_executors.cc:857 (memcached+0x000000501d8e)
|
#13 execute_request_packet(Cookie&, cb::mcbp::Request const&) kv_engine/daemon/mcbp_executors.cc:881 (memcached+0x000000501f67)
|
#14 Cookie::execute() kv_engine/daemon/cookie.cc:124 (memcached+0x0000004e507b)
|
#15 StateMachine::conn_execute() kv_engine/daemon/statemachine.cc:410 (memcached+0x000000537eec)
|
#16 StateMachine::execute() kv_engine/daemon/statemachine.cc:137 (memcached+0x00000053a227)
|
#17 Connection::runStateMachinery() kv_engine/daemon/connection.cc:1377 (memcached+0x0000004cb3f7)
|
#18 Connection::runEventLoop(short) kv_engine/daemon/connection.cc:1451 (memcached+0x0000004cdbfe)
|
#19 run_event_loop(Connection*, short) kv_engine/daemon/connections.cc:148 (memcached+0x0000004e349b)
|
#20 event_handler(int, short, void*) kv_engine/daemon/memcached.cc:840 (memcached+0x000000432d2d)
|
#21 event_persist_closure /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1580 (libevent_core.so.2.1.8+0x000000017086)
|
#22 event_process_active_single_queue /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1639 (libevent_core.so.2.1.8+0x000000017086)
|
#23 CouchbaseThread::run() platform/src/cb_pthreads.cc:58 (libplatform_so.so.0.1.0+0x000000009c5f)
|
#24 platform_thread_wrap platform/src/cb_pthreads.cc:71 (libplatform_so.so.0.1.0+0x000000009c5f)
|
#25 (libtsan.so.0+0x000000024feb)
|
Logs: http://cv.jenkins.couchbase.com/job/kv_engine-master-post-commit-TSan/369/ThreadSanitizer/