Details
-
Bug
-
Resolution: Fixed
-
Critical
-
Cheshire-Cat
-
Triaged
-
1
-
Unknown
-
KV-Engine 2021-Jan
Description
After upgrading TSan CV job to use clang-9 (from GCC 7.3), the following use-after-free bug is reported:
WARNING: ThreadSanitizer: heap-use-after-free (pid=2802)
|
Atomic read of size 1 at 0x7b18002aed33 by thread T9 (mutexes: read M1062984889428614268, write M755133):
|
#0 __tsan_atomic8_load <null> (ep_testsuite_dcp+0x4eb76c)
|
#1 std::__atomic_base<unsigned char>::load(std::memory_order) const /usr/bin/../lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/c++/7.3.0/bits/atomic_base.h:396:9 (libep.so+0x13f4da)
|
#2 folly::AtomicBitSet<1ul>::test(unsigned long, std::memory_order) const kv_engine/include/memcached/3rd_party/folly/AtomicBitSet.h:160:35 (libep.so+0x13f4da)
|
#3 StoredValue::isOrdered() const kv_engine/engines/ep/src/stored-value.h:902:21 (libep.so+0x1408e3)
|
#4 StoredValue::key() kv_engine/engines/ep/src/stored-value.h:1196:9 (libep.so+0x140885)
|
#5 StoredValue::getKey() const kv_engine/engines/ep/src/stored-value.h:275:49 (libep.so+0x13f375)
|
#6 EphemeralVBucket::pageOut(Collections::VB::ReadHandle const&, HashTable::HashBucketLock const&, StoredValue*&) kv_engine/engines/ep/src/ephemeral_vb.cc:123:43 (libep.so+0x332e3a)
|
#7 PagingVisitor::doEviction(HashTable::HashBucketLock const&, StoredValue*) kv_engine/engines/ep/src/paging_visitor.cc:364:24 (libep.so+0x3c8710)
|
#8 PagingVisitor::visit(HashTable::HashBucketLock const&, StoredValue&) kv_engine/engines/ep/src/paging_visitor.cc:157:14 (libep.so+0x3c84ad)
|
#9 non-virtual thunk to PagingVisitor::visit(HashTable::HashBucketLock const&, StoredValue&) kv_engine/engines/ep/src/paging_visitor.cc (libep.so+0x3c87c9)
|
#10 HashTable::pauseResumeVisit(HashTableVisitor&, HashTable::Position&) kv_engine/engines/ep/src/hash_table.cc:1222:39 (libep.so+0x3662c8)
|
#11 HashTable::visit(HashTableVisitor&) kv_engine/engines/ep/src/hash_table.cc:1119:18 (libep.so+0x365fa0)
|
#12 PagingVisitor::visitBucket(std::shared_ptr<VBucket> const&) kv_engine/engines/ep/src/paging_visitor.cc:255:20 (libep.so+0x3c8afb)
|
#13 VBCBAdaptor::run() kv_engine/engines/ep/src/kv_bucket.cc:2379:22 (libep.so+0x392b98)
|
#14 GlobalTask::execute() kv_engine/engines/ep/src/globaltask.cc:73:12 (libep.so+0x361069)
|
#15 CB3ExecutorThread::run() kv_engine/engines/ep/src/cb3_executorthread.cc:174:39 (libep.so+0x187dc4)
|
#16 launch_executor_thread(void*) kv_engine/engines/ep/src/cb3_executorthread.cc:34:15 (libep.so+0x187795)
|
#17 CouchbaseThread::run() platform/src/cb_pthreads.cc:58:9 (libplatform_so.so.0.1.0+0x2c0ba)
|
#18 platform_thread_wrap(void*) platform/src/cb_pthreads.cc:71:14 (libplatform_so.so.0.1.0+0x2a9fe)
|
|
Previous write of size 8 at 0x7b18002aed30 by thread T18:
|
#0 operator delete(void*) <null> (ep_testsuite_dcp+0x530a7e)
|
#1 OrderedStoredValue::operator delete(void*) kv_engine/engines/ep/src/stored-value.h:1120:9 (libep.so+0x3d3255)
|
#2 std::default_delete<OrderedStoredValue>::operator()(OrderedStoredValue*) const /usr/bin/../lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/c++/7.3.0/bits/unique_ptr.h:78:2 (libep.so+0x3d5ce2)
|
#3 std::unique_ptr<OrderedStoredValue, std::default_delete<OrderedStoredValue> >::~unique_ptr() /usr/bin/../lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/c++/7.3.0/bits/unique_ptr.h:268:4 (libep.so+0x3d0c17)
|
#4 BasicLinkedList::purgeListElem(boost::intrusive::list_iterator<boost::intrusive::mhtraits<OrderedStoredValue, boost::intrusive::list_member_hook<void, void, void>, &(OrderedStoredValue::seqno_hook)>, false>, bool) kv_engine/engines/ep/src/linked_list.cc:529:1 (libep.so+0x3cdf23)
|
#5 BasicLinkedList::purgeTombstones(long, std::function<bool (DocKey const&, long, bool)>, std::function<bool ()>) kv_engine/engines/ep/src/linked_list.cc:392:18 (libep.so+0x3cdb86)
|
#6 EphemeralVBucket::purgeStaleItems(std::function<bool ()>) kv_engine/engines/ep/src/ephemeral_vb.cc:373:35 (libep.so+0x333e42)
|
#7 EphemeralVBucket::StaleItemDeleter::visit(VBucket&) kv_engine/engines/ep/src/ephemeral_tombstone_purger.cc:205:37 (libep.so+0x33199e)
|
#8 KVBucket::pauseResumeVisit(PauseResumeVBVisitor&, KVBucketIface::Position&) kv_engine/engines/ep/src/kv_bucket.cc:2322:36 (libep.so+0x392648)
|
#9 EphTombstoneStaleItemDeleter::run() kv_engine/engines/ep/src/ephemeral_tombstone_purger.cc:274:20 (libep.so+0x32e15c)
|
#10 GlobalTask::execute() kv_engine/engines/ep/src/globaltask.cc:73:12 (libep.so+0x361069)
|
#11 CB3ExecutorThread::run() kv_engine/engines/ep/src/cb3_executorthread.cc:174:39 (libep.so+0x187dc4)
|
#12 launch_executor_thread(void*) kv_engine/engines/ep/src/cb3_executorthread.cc:34:15 (libep.so+0x187795)
|
#13 CouchbaseThread::run() platform/src/cb_pthreads.cc:58:9 (libplatform_so.so.0.1.0+0x2c0ba)
|
#14 platform_thread_wrap(void*) platform/src/cb_pthreads.cc:71:14 (libplatform_so.so.0.1.0+0x2a9fe)
|
|
Mutex M1062984889428614268 is already destroyed.
|
|
Mutex M755133 (0x7b7000002990) created at:
|
#0 pthread_mutex_lock <null> (ep_testsuite_dcp+0x4bfaa6)
|
#1 __gthread_mutex_lock(pthread_mutex_t*) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/x86_64-linux-gnu/c++/7.3.0/bits/gthr-default.h:748:12 (ep_testsuite_dcp+0x548bc3)
|
#2 std::mutex::lock() /usr/bin/../lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/c++/7.3.0/bits/std_mutex.h:103:17 (ep_testsuite_dcp+0x54e055)
|
#3 std::unique_lock<std::mutex>::lock() /usr/bin/../lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/c++/7.3.0/bits/std_mutex.h:267:17 (ep_testsuite_dcp+0x5cd58b)
|
#4 std::unique_lock<std::mutex>::unique_lock(std::mutex&) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/c++/7.3.0/bits/std_mutex.h:197:2 (ep_testsuite_dcp+0x5cd4c1)
|
#5 HashTable::HashBucketLock::HashBucketLock(int, std::mutex&) kv_engine/engines/ep/src/hash_table.h:341:37 (libep.so+0x1f9a1e)
|
#6 HashTable::getLockedBucketForHash(int) kv_engine/engines/ep/src/hash_table.h:1278:28 (libep.so+0x1f97af)
|
#7 HashTable::getLockedBucket(DocKey const&) kv_engine/engines/ep/src/hash_table.h:1056:16 (libep.so+0x1f8758)
|
#8 HashTable::findInner(DocKey const&) kv_engine/engines/ep/src/hash_table.cc:315:26 (libep.so+0x363080)
|
#9 HashTable::findForUpdate(DocKey const&) kv_engine/engines/ep/src/hash_table.cc:950:19 (libep.so+0x364fae)
|
#10 VBucket::set(Item&, void const*, EventuallyPersistentEngine&, std::function<cb::StoreIfStatus (std::optional<item_info> const&, cb::vbucket_info)>, Collections::VB::CachingReadHandle const&) kv_engine/engines/ep/src/vbucket.cc:1558:25 (libep.so+0x3fd9fc)
|
#11 KVBucket::set(Item&, void const*, std::function<cb::StoreIfStatus (std::optional<item_info> const&, cb::vbucket_info)>) kv_engine/engines/ep/src/kv_bucket.cc:694:22 (libep.so+0x389802)
|
#12 EventuallyPersistentEngine::storeIfInner(void const*, Item&, unsigned long, ENGINE_STORE_OPERATION, std::function<cb::StoreIfStatus (std::optional<item_info> const&, cb::vbucket_info)> const&, bool) kv_engine/engines/ep/src/ep_engine.cc:2651:28 (libep.so+0x2cffa8)
|
#13 EventuallyPersistentEngine::storeInner(void const*, Item&, unsigned long&, ENGINE_STORE_OPERATION, bool) kv_engine/engines/ep/src/ep_engine.cc:2714:15 (libep.so+0x2cfb0f)
|
#14 EventuallyPersistentEngine::store(gsl::not_null<void const*>, gsl::not_null<void*>, unsigned long&, ENGINE_STORE_OPERATION, std::optional<cb::durability::Requirements> const&, DocumentState, bool) kv_engine/engines/ep/src/ep_engine.cc:444:33 (libep.so+0x2cfa64)
|
#15 ENGINE_ERROR_CODE std::__invoke_impl<ENGINE_ERROR_CODE, ENGINE_ERROR_CODE (EngineIface::*&)(gsl::not_null<void const*>, gsl::not_null<void*>, unsigned long&, ENGINE_STORE_OPERATION, std::optional<cb::durability::Requirements> const&, DocumentState, bool), EngineIface*&, gsl::not_null<void const*>&, gsl::not_null<void*>&, unsigned long&, ENGINE_STORE_OPERATION&, std::optional<cb::durability::Requirements>&, DocumentState&, bool&>(std::__invoke_memfun_deref, ENGINE_ERROR_CODE (EngineIface::*&)(gsl::not_null<void const*>, gsl::not_null<void*>, unsigned long&, ENGINE_STORE_OPERATION, std::optional<cb::durability::Requirements> const&, DocumentState, bool), EngineIface*&, gsl::not_null<void const*>&, gsl::not_null<void*>&, unsigned long&, ENGINE_STORE_OPERATION&, std::optional<cb::durability::Requirements>&, DocumentState&, bool&) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/c++/7.3.0/bits/invoke.h:73:14 (ep_testsuite_dcp+0x61c4ac)
|
#16 std::__invoke_result<ENGINE_ERROR_CODE (EngineIface::*&)(gsl::not_null<void const*>, gsl::not_null<void*>, unsigned long&, ENGINE_STORE_OPERATION, std::optional<cb::durability::Requirements> const&, DocumentState, bool), EngineIface*&, gsl::not_null<void const*>&, gsl::not_null<void*>&, unsigned long&, ENGINE_STORE_OPERATION&, std::optional<cb::durability::Requirements>&, DocumentState&, bool&>::type std::__invoke<ENGINE_ERROR_CODE (EngineIface::*&)(gsl::not_null<void const*>, gsl::not_null<void*>, unsigned long&, ENGINE_STORE_OPERATION, std::optional<cb::durability::Requirements> const&, DocumentState, bool), EngineIface*&, gsl::not_null<void const*>&, gsl::not_null<void*>&, unsigned long&, ENGINE_STORE_OPERATION&, std::optional<cb::durability::Requirements>&, DocumentState&, bool&>(ENGINE_ERROR_CODE (EngineIface::*&)(gsl::not_null<void const*>, gsl::not_null<void*>, unsigned long&, ENGINE_STORE_OPERATION, std::optional<cb::durability::Requirements> const&, DocumentState, bool), EngineIface*&, gsl::not_null<void const*>&, gsl::not_null<void*>&, unsigned long&, ENGINE_STORE_OPERATION&, std::optional<cb::durability::Requirements>&, DocumentState&, bool&) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/c++/7.3.0/bits/invoke.h:95:14 (ep_testsuite_dcp+0x61c0fc)
|
#17 ENGINE_ERROR_CODE std::_Bind<ENGINE_ERROR_CODE (EngineIface::* (EngineIface*, gsl::not_null<void const*>, gsl::not_null<void*>, std::reference_wrapper<unsigned long>, ENGINE_STORE_OPERATION, std::optional<cb::durability::Requirements>, DocumentState, bool))(gsl::not_null<void const*>, gsl::not_null<void*>, unsigned long&, ENGINE_STORE_OPERATION, std::optional<cb::durability::Requirements> const&, DocumentState, bool)>::__call<ENGINE_ERROR_CODE, 0ul, 1ul, 2ul, 3ul, 4ul, 5ul, 6ul, 7ul>(std::tuple<>&&, std::_Index_tuple<0ul, 1ul, 2ul, 3ul, 4ul, 5ul, 6ul, 7ul>) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/c++/7.3.0/functional:467:11 (ep_testsuite_dcp+0x61c00b)
|
#18 ENGINE_ERROR_CODE std::_Bind<ENGINE_ERROR_CODE (EngineIface::* (EngineIface*, gsl::not_null<void const*>, gsl::not_null<void*>, std::reference_wrapper<unsigned long>, ENGINE_STORE_OPERATION, std::optional<cb::durability::Requirements>, DocumentState, bool))(gsl::not_null<void const*>, gsl::not_null<void*>, unsigned long&, ENGINE_STORE_OPERATION, std::optional<cb::durability::Requirements> const&, DocumentState, bool)>::operator()<ENGINE_ERROR_CODE>() /usr/bin/../lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/c++/7.3.0/functional:549:17 (ep_testsuite_dcp+0x61be83)
|
#19 std::_Function_handler<ENGINE_ERROR_CODE (), std::_Bind<ENGINE_ERROR_CODE (EngineIface::* (EngineIface*, gsl::not_null<void const*>, gsl::not_null<void*>, std::reference_wrapper<unsigned long>, ENGINE_STORE_OPERATION, std::optional<cb::durability::Requirements>, DocumentState, bool))(gsl::not_null<void const*>, gsl::not_null<void*>, unsigned long&, ENGINE_STORE_OPERATION, std::optional<cb::durability::Requirements> const&, DocumentState, bool)> >::_M_invoke(std::_Any_data const&) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/c++/7.3.0/bits/std_function.h:301:9 (ep_testsuite_dcp+0x61b3ed)
|
#20 std::function<ENGINE_ERROR_CODE ()>::operator()() const /usr/bin/../lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/c++/7.3.0/bits/std_function.h:706:14 (ep_testsuite_dcp+0x60bc08)
|
#21 call_engine_and_handle_EWOULDBLOCK(MockCookie*, std::function<ENGINE_ERROR_CODE ()> const&) kv_engine/programs/engine_testapp/mock_engine.cc:61:19 (ep_testsuite_dcp+0x606077)
|
#22 MockEngine::store(gsl::not_null<void const*>, gsl::not_null<void*>, unsigned long&, ENGINE_STORE_OPERATION, std::optional<cb::durability::Requirements> const&, DocumentState, bool) kv_engine/programs/engine_testapp/mock_engine.cc:343:12 (ep_testsuite_dcp+0x606dd5)
|
#23 storeCasVb11(EngineIface*, void const*, ENGINE_STORE_OPERATION, char const*, char const*, unsigned long, unsigned int, unsigned long, Vbid, unsigned int, unsigned char, DocumentState, std::optional<cb::durability::Requirements> const&) kv_engine/engines/ep/tests/ep_test_apis.cc:1205:24 (ep_testsuite_dcp+0x5419d8)
|
#24 store(EngineIface*, void const*, ENGINE_STORE_OPERATION, char const*, char const*, void**, unsigned long, Vbid, unsigned int, unsigned char, DocumentState, std::optional<cb::durability::Requirements> const&) kv_engine/engines/ep/tests/ep_test_apis.cc:1116:16 (ep_testsuite_dcp+0x5437a7)
|
#25 write_items_upto_mem_perc(EngineIface*, int, int, char const*, char const*) kv_engine/engines/ep/tests/ep_test_apis.cc:1858:17 (ep_testsuite_dcp+0x54873e)
|
#26 test_dcp_cursor_dropping_backfill(EngineIface*) kv_engine/engines/ep/tests/ep_testsuite_dcp.cc:3447:18 (ep_testsuite_dcp+0x5a424d)
|
#27 execute_test(test, char const*, char const*) kv_engine/programs/engine_testapp/engine_testapp.cc:378:19 (ep_testsuite_dcp+0x55c1da)
|
#28 main kv_engine/programs/engine_testapp/engine_testapp.cc:629:37 (ep_testsuite_dcp+0x55b5db)
|
|
Thread T9 'mc:NonIO_3' (tid=26331, running) created by main thread at:
|
#0 pthread_create <null> (ep_testsuite_dcp+0x4a184b)
|
#1 cb_create_named_thread(unsigned long*, void (*)(void*), void*, int, char const*) platform/src/cb_pthreads.cc:109:11 (libplatform_so.so.0.1.0+0x2a94e)
|
#2 CB3ExecutorThread::start() kv_engine/engines/ep/src/cb3_executorthread.cc:51:9 (libep.so+0x18760c)
|
#3 CB3ExecutorPool::_adjustWorkers(task_type_t, unsigned long) kv_engine/engines/ep/src/cb3_executorpool.cc:464:33 (libep.so+0x164031)
|
#4 CB3ExecutorPool::_startWorkers() kv_engine/engines/ep/src/cb3_executorpool.cc:528:5 (libep.so+0x1646c2)
|
#5 CB3ExecutorPool::_registerTaskable(Taskable&) kv_engine/engines/ep/src/cb3_executorpool.cc:419:5 (libep.so+0x163d76)
|
#6 CB3ExecutorPool::registerTaskable(Taskable&) kv_engine/engines/ep/src/cb3_executorpool.cc:424:5 (libep.so+0x163de6)
|
#7 KVBucket::KVBucket(EventuallyPersistentEngine&) kv_engine/engines/ep/src/kv_bucket.cc:323:26 (libep.so+0x386e0d)
|
#8 EphemeralBucket::EphemeralBucket(EventuallyPersistentEngine&) kv_engine/engines/ep/src/ephemeral_bucket.cc:96:7 (libep.so+0x324cf7)
|
#9 std::_MakeUniq<EphemeralBucket>::__single_object std::make_unique<EphemeralBucket, EventuallyPersistentEngine&>(EventuallyPersistentEngine&) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/c++/7.3.0/bits/unique_ptr.h:825:34 (libep.so+0x2eff14)
|
#10 EventuallyPersistentEngine::makeBucket(Configuration&) kv_engine/engines/ep/src/ep_engine.cc:6540:16 (libep.so+0x2d92e3)
|
#11 EventuallyPersistentEngine::initialize(char const*) kv_engine/engines/ep/src/ep_engine.cc:2248:16 (libep.so+0x2d8d59)
|
#12 MockTestHarness::create_bucket(bool, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) kv_engine/programs/engine_testapp/engine_testapp.cc:212:52 (ep_testsuite_dcp+0x55d1b7)
|
#13 execute_test(test, char const*, char const*) kv_engine/programs/engine_testapp/engine_testapp.cc:366:43 (ep_testsuite_dcp+0x55c155)
|
#14 main kv_engine/programs/engine_testapp/engine_testapp.cc:629:37 (ep_testsuite_dcp+0x55b5db)
|
|
Thread T18 'mc:NonIO_2' (tid=26330, running) created by main thread at:
|
#0 pthread_create <null> (ep_testsuite_dcp+0x4a184b)
|
#1 cb_create_named_thread(unsigned long*, void (*)(void*), void*, int, char const*) platform/src/cb_pthreads.cc:109:11 (libplatform_so.so.0.1.0+0x2a94e)
|
#2 CB3ExecutorThread::start() kv_engine/engines/ep/src/cb3_executorthread.cc:51:9 (libep.so+0x18760c)
|
#3 CB3ExecutorPool::_adjustWorkers(task_type_t, unsigned long) kv_engine/engines/ep/src/cb3_executorpool.cc:464:33 (libep.so+0x164031)
|
#4 CB3ExecutorPool::_startWorkers() kv_engine/engines/ep/src/cb3_executorpool.cc:528:5 (libep.so+0x1646c2)
|
#5 CB3ExecutorPool::_registerTaskable(Taskable&) kv_engine/engines/ep/src/cb3_executorpool.cc:419:5 (libep.so+0x163d76)
|
#6 CB3ExecutorPool::registerTaskable(Taskable&) kv_engine/engines/ep/src/cb3_executorpool.cc:424:5 (libep.so+0x163de6)
|
#7 KVBucket::KVBucket(EventuallyPersistentEngine&) kv_engine/engines/ep/src/kv_bucket.cc:323:26 (libep.so+0x386e0d)
|
#8 EphemeralBucket::EphemeralBucket(EventuallyPersistentEngine&) kv_engine/engines/ep/src/ephemeral_bucket.cc:96:7 (libep.so+0x324cf7)
|
#9 std::_MakeUniq<EphemeralBucket>::__single_object std::make_unique<EphemeralBucket, EventuallyPersistentEngine&>(EventuallyPersistentEngine&) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/c++/7.3.0/bits/unique_ptr.h:825:34 (libep.so+0x2eff14)
|
#10 EventuallyPersistentEngine::makeBucket(Configuration&) kv_engine/engines/ep/src/ep_engine.cc:6540:16 (libep.so+0x2d92e3)
|
#11 EventuallyPersistentEngine::initialize(char const*) kv_engine/engines/ep/src/ep_engine.cc:2248:16 (libep.so+0x2d8d59)
|
#12 MockTestHarness::create_bucket(bool, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) kv_engine/programs/engine_testapp/engine_testapp.cc:212:52 (ep_testsuite_dcp+0x55d1b7)
|
#13 execute_test(test, char const*, char const*) kv_engine/programs/engine_testapp/engine_testapp.cc:366:43 (ep_testsuite_dcp+0x55c155)
|
#14 main kv_engine/programs/engine_testapp/engine_testapp.cc:629:37 (ep_testsuite_dcp+0x55b5db)
|
|
SUMMARY: ThreadSanitizer: heap-use-after-free (/home/couchbase/jenkins/workspace/kv_engine.threadsanitizer_master/build/kv_engine/ep_testsuite_dcp+0x4eb76c) in __tsan_atomic8_load
|