Details
-
Bug
-
Resolution: Fixed
-
Major
-
Cheshire-Cat
-
Untriaged
-
Unknown
-
Magma 2020-Oct1
Description
Heap use-after-free issue when running ep_testsuite_basic.value_eviction.magma, test vbucket deletion doesn't affect new data seen during CV:
Running [0036/0047]: vbucket deletion doesn't affect new data...==================
|
WARNING: ThreadSanitizer: heap-use-after-free (pid=58911)
|
Atomic read of size 1 at 0x7b200001ae40 by thread T23:
|
#0 pthread_mutex_lock <null> (libtsan.so.0+0x00000003876f)
|
#1 __gthread_mutex_lock /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:748 (libmagma.so+0x00000013825f)
|
#2 std::mutex::lock() /usr/local/include/c++/7.3.0/bits/std_mutex.h:103 (libmagma.so+0x00000013825f)
|
#3 std::unique_lock<std::mutex>::lock() /usr/local/include/c++/7.3.0/bits/std_mutex.h:267 (libmagma.so+0x00000013825f)
|
#4 std::unique_lock<std::mutex>::unique_lock(std::mutex&) /usr/local/include/c++/7.3.0/bits/std_mutex.h:197 (libmagma.so+0x00000013825f)
|
#5 magma::PersistentSnapshotManager::completeTransaction() ../magma/lsm/persistent_state.cc:426 (libmagma.so+0x00000013825f)
|
#6 magma::PersistTransaction::Reset() ../magma/lsm/persistent_state.h:123 (libmagma.so+0x000000101d7a)
|
#7 magma::PersistTransaction::~PersistTransaction() ../magma/lsm/persistent_state.h:129 (libmagma.so+0x000000101d7a)
|
#8 magma::LSMTree::FlushWork::~FlushWork() ../magma/lsm/lsm_tree.h:162 (libmagma.so+0x000000101d7a)
|
#9 void __gnu_cxx::new_allocator<magma::LSMTree::FlushWork>::destroy<magma::LSMTree::FlushWork>(magma::LSMTree::FlushWork*) /usr/local/include/c++/7.3.0/ext/new_allocator.h:140 (libmagma.so+0x000000101d7a)
|
#10 void std::allocator_traits<std::allocator<magma::LSMTree::FlushWork> >::destroy<magma::LSMTree::FlushWork>(std::allocator<magma::LSMTree::FlushWork>&, magma::LSMTree::FlushWork*) /usr/local/include/c++/7.3.0/bits/alloc_traits.h:487 (libmagma.so+0x000000101d7a)
|
#11 std::_Sp_counted_ptr_inplace<magma::LSMTree::FlushWork, std::allocator<magma::LSMTree::FlushWork>, (__gnu_cxx::_Lock_policy)2>::_M_dispose() /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:535 (libmagma.so+0x000000101d7a)
|
#12 std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release() /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:154 (ep_testsuite_basic+0x0000004454dd)
|
#13 std::__shared_count<(__gnu_cxx::_Lock_policy)2>::~__shared_count() /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:684 (libmagma.so+0x0000000f4859)
|
#14 std::__shared_ptr<magma::LSMTree::FlushWork, (__gnu_cxx::_Lock_policy)2>::~__shared_ptr() /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:1123 (libmagma.so+0x0000000f4859)
|
#15 std::shared_ptr<magma::LSMTree::FlushWork>::~shared_ptr() /usr/local/include/c++/7.3.0/bits/shared_ptr.h:93 (libmagma.so+0x0000000f4859)
|
#16 ~<lambda> ../magma/magma/kvstore.cc:181 (libmagma.so+0x0000000f4859)
|
#17 _M_destroy /usr/local/include/c++/7.3.0/bits/std_function.h:207 (libmagma.so+0x0000000f4859)
|
#18 _M_manager /usr/local/include/c++/7.3.0/bits/std_function.h:231 (libmagma.so+0x0000000f4859)
|
#19 std::_Function_base::~_Function_base() /usr/local/include/c++/7.3.0/bits/std_function.h:276 (libmagma.so+0x0000000aa144)
|
#20 std::function<void ()>::~function() /usr/local/include/c++/7.3.0/bits/std_function.h:389 (libmagma.so+0x0000000aa144)
|
#21 magma::Task::~Task() ../magma/util/worker.h:33 (libmagma.so+0x0000000aa144)
|
#22 magma::TaskWorker::loop(void*) ../magma/util/worker.cc:85 (libmagma.so+0x0000000aa144)
|
#23 CouchbaseThread::run() ../platform/src/cb_pthreads.cc:58 (libplatform_so.so.0.1.0+0x000000010c6b)
|
#24 platform_thread_wrap ../platform/src/cb_pthreads.cc:71 (libplatform_so.so.0.1.0+0x000000010c6b)
|
#25 <null> <null> (libtsan.so.0+0x000000024feb)
|
|
Previous write of size 8 at 0x7b200001ae40 by main thread (mutexes: write M919995927677109360):
|
#0 free <null> (libtsan.so.0+0x000000027806)
|
#1 cb::SystemArenaMalloc::free(void*) ../platform/src/system_arena_malloc.cc:121 (libplatform_so.so.0.1.0+0x000000024b3b)
|
#2 cb::SystemArenaMalloc::sized_free(void*, unsigned long) ../platform/src/system_arena_malloc.cc:125 (libplatform_so.so.0.1.0+0x000000024b69)
|
#3 cb::_ArenaMalloc<cb::SystemArenaMalloc>::sized_free(void*, unsigned long) ../platform/include/platform/cb_arena_malloc.h:169 (ep_testsuite_basic+0x0000004cb74a)
|
#4 cb_sized_free ../platform/src/cb_malloc_arena.cc:71 (ep_testsuite_basic+0x0000004cb74a)
|
#5 operator delete(void*, unsigned long) ../platform/src/global_new_replacement.cc:92 (ep_testsuite_basic+0x0000004cb49f)
|
#6 std::default_delete<magma::PersistentSnapshotManager>::operator()(magma::PersistentSnapshotManager*) const /usr/local/include/c++/7.3.0/bits/unique_ptr.h:78 (libmagma.so+0x00000011dc74)
|
#7 std::unique_ptr<magma::PersistentSnapshotManager, std::default_delete<magma::PersistentSnapshotManager> >::~unique_ptr() /usr/local/include/c++/7.3.0/bits/unique_ptr.h:268 (libmagma.so+0x00000011dc74)
|
#8 magma::LSMTree::~LSMTree() ../magma/lsm/lsm_tree.cc:337 (libmagma.so+0x00000011dc74)
|
#9 std::default_delete<magma::LSMTree>::operator()(magma::LSMTree*) const /usr/local/include/c++/7.3.0/bits/unique_ptr.h:78 (libmagma.so+0x0000000f6acd)
|
#10 std::unique_ptr<magma::LSMTree, std::default_delete<magma::LSMTree> >::reset(magma::LSMTree*) /usr/local/include/c++/7.3.0/bits/unique_ptr.h:376 (libmagma.so+0x0000000f6acd)
|
#11 magma::KVStore::~KVStore() ../magma/magma/kvstore.cc:276 (libmagma.so+0x0000000f6acd)
|
#12 void __gnu_cxx::new_allocator<magma::KVStore>::destroy<magma::KVStore>(magma::KVStore*) /usr/local/include/c++/7.3.0/ext/new_allocator.h:140 (libmagma.so+0x0000000e4e00)
|
#13 void std::allocator_traits<std::allocator<magma::KVStore> >::destroy<magma::KVStore>(std::allocator<magma::KVStore>&, magma::KVStore*) /usr/local/include/c++/7.3.0/bits/alloc_traits.h:487 (libmagma.so+0x0000000e4e00)
|
#14 std::_Sp_counted_ptr_inplace<magma::KVStore, std::allocator<magma::KVStore>, (__gnu_cxx::_Lock_policy)2>::_M_dispose() /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:535 (libmagma.so+0x0000000e4e00)
|
#15 std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release() /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:154 (ep_testsuite_basic+0x0000004454dd)
|
#16 std::__shared_count<(__gnu_cxx::_Lock_policy)2>::~__shared_count() /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:684 (libmagma.so+0x0000000e4357)
|
#17 std::__shared_ptr<magma::KVStore, (__gnu_cxx::_Lock_policy)2>::~__shared_ptr() /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:1123 (libmagma.so+0x0000000e4357)
|
#18 std::__shared_ptr<magma::KVStore, (__gnu_cxx::_Lock_policy)2>::reset() /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:1235 (libmagma.so+0x0000000e4357)
|
#19 magma::Magma::~Magma() ../magma/magma/magma.cc:64 (libmagma.so+0x0000000e4357)
|
#20 std::default_delete<magma::Magma>::operator()(magma::Magma*) const /usr/local/include/c++/7.3.0/bits/unique_ptr.h:78 (libep.so+0x00000032ca7e)
|
#21 std::unique_ptr<magma::Magma, std::default_delete<magma::Magma> >::~unique_ptr() /usr/local/include/c++/7.3.0/bits/unique_ptr.h:268 (libep.so+0x00000032ca7e)
|
#22 MagmaKVStore::~MagmaKVStore() ../kv_engine/engines/ep/src/magma-kvstore/magma-kvstore.cc:630 (libep.so+0x00000032ca7e)
|
#23 MagmaKVStore::~MagmaKVStore() ../kv_engine/engines/ep/src/magma-kvstore/magma-kvstore.cc:635 (libep.so+0x00000032cadd)
|
#24 std::default_delete<KVStore>::operator()(KVStore*) const /usr/local/include/c++/7.3.0/bits/unique_ptr.h:78 (libep.so+0x0000002483fe)
|
#25 std::unique_ptr<KVStore, std::default_delete<KVStore> >::~unique_ptr() /usr/local/include/c++/7.3.0/bits/unique_ptr.h:268 (libep.so+0x0000002483fe)
|
#26 KVShard::~KVShard() ../kv_engine/engines/ep/src/kvshard.h:75 (libep.so+0x0000002483fe)
|
#27 std::default_delete<KVShard>::operator()(KVShard*) const /usr/local/include/c++/7.3.0/bits/unique_ptr.h:78 (libep.so+0x000000234878)
|
#28 std::unique_ptr<KVShard, std::default_delete<KVShard> >::~unique_ptr() /usr/local/include/c++/7.3.0/bits/unique_ptr.h:268 (libep.so+0x000000234878)
|
#29 void std::_Destroy<std::unique_ptr<KVShard, std::default_delete<KVShard> > >(std::unique_ptr<KVShard, std::default_delete<KVShard> >*) /usr/local/include/c++/7.3.0/bits/stl_construct.h:98 (libep.so+0x000000234878)
|
#30 void std::_Destroy_aux<false>::__destroy<std::unique_ptr<KVShard, std::default_delete<KVShard> >*>(std::unique_ptr<KVShard, std::default_delete<KVShard> >*, std::unique_ptr<KVShard, std::default_delete<KVShard> >*) /usr/local/include/c++/7.3.0/bits/stl_construct.h:108 (libep.so+0x000000234878)
|
#31 void std::_Destroy<std::unique_ptr<KVShard, std::default_delete<KVShard> >*>(std::unique_ptr<KVShard, std::default_delete<KVShard> >*, std::unique_ptr<KVShard, std::default_delete<KVShard> >*) /usr/local/include/c++/7.3.0/bits/stl_construct.h:137 (libep.so+0x000000234878)
|
#32 void std::_Destroy<std::unique_ptr<KVShard, std::default_delete<KVShard> >*, std::unique_ptr<KVShard, std::default_delete<KVShard> > >(std::unique_ptr<KVShard, std::default_delete<KVShard> >*, std::unique_ptr<KVShard, std::default_delete<KVShard> >*, std::allocator<std::unique_ptr<KVShard, std::default_delete<KVShard> > >&) /usr/local/include/c++/7.3.0/bits/stl_construct.h:206 (libep.so+0x000000234878)
|
#33 std::vector<std::unique_ptr<KVShard, std::default_delete<KVShard> >, std::allocator<std::unique_ptr<KVShard, std::default_delete<KVShard> > > >::~vector() /usr/local/include/c++/7.3.0/bits/stl_vector.h:434 (libep.so+0x000000234878)
|
#34 VBucketMap::~VBucketMap() ../kv_engine/engines/ep/src/vbucketmap.h:32 (libep.so+0x000000234878)
|
#35 KVBucket::~KVBucket() ../kv_engine/engines/ep/src/kv_bucket.cc:471 (libep.so+0x000000234878)
|
#36 EPBucket::~EPBucket() ../kv_engine/engines/ep/src/ep_bucket.cc:280 (libep.so+0x000000160954)
|
#37 EPBucket::~EPBucket() ../kv_engine/engines/ep/src/ep_bucket.cc:281 (libep.so+0x000000160977)
|
#38 std::default_delete<KVBucket>::operator()(KVBucket*) const /usr/local/include/c++/7.3.0/bits/unique_ptr.h:78 (libep.so+0x0000001a442d)
|
#39 std::unique_ptr<KVBucket, std::default_delete<KVBucket> >::reset(KVBucket*) /usr/local/include/c++/7.3.0/bits/unique_ptr.h:376 (libep.so+0x0000001a442d)
|
#40 EventuallyPersistentEngine::~EventuallyPersistentEngine() ../kv_engine/engines/ep/src/ep_engine.cc:6628 (libep.so+0x0000001a442d)
|
#41 EventuallyPersistentEngine::~EventuallyPersistentEngine() ../kv_engine/engines/ep/src/ep_engine.cc:6648 (libep.so+0x0000001a46ed)
|
#42 EventuallyPersistentEngine::destroy(bool) ../kv_engine/engines/ep/src/ep_engine.cc:218 (libep.so+0x000000187075)
|
#43 MockEngine::destroy(bool) ../kv_engine/programs/engine_testapp/mock_engine.cc:136 (ep_testsuite_basic+0x0000004cc47a)
|
#44 MockTestHarness::destroy_bucket(EngineIface*, bool) ../kv_engine/programs/engine_testapp/engine_testapp.cc:226 (ep_testsuite_basic+0x000000441ad8)
|
#45 execute_test ../kv_engine/programs/engine_testapp/engine_testapp.cc:403 (ep_testsuite_basic+0x000000441ad8)
|
#46 main ../kv_engine/programs/engine_testapp/engine_testapp.cc:624 (ep_testsuite_basic+0x000000442eed)
|
|
Mutex M919995927677109360 is already destroyed.
|
|
Thread T23 'mg:flusher:0:de' (tid=84002, running) created by main thread at:
|
#0 pthread_create <null> (libtsan.so.0+0x0000000282a0)
|
#1 cb_create_named_thread(unsigned long*, void (*)(void*), void*, int, char const*) ../platform/src/cb_pthreads.cc:109 (libplatform_so.so.0.1.0+0x0000000112a4)
|
#2 magma::TaskWorker::Start() ../magma/util/worker.cc:72 (libmagma.so+0x0000000a980b)
|
#3 magma::ThreadPool::ThreadPool(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long, unsigned long) ../magma/util/thread_pool.cc:37 (libmagma.so+0x0000000abd40)
|
#4 void __gnu_cxx::new_allocator<magma::ThreadPool>::construct<magma::ThreadPool, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long&, unsigned long&>(magma::ThreadPool*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long&, unsigned long&) /usr/local/include/c++/7.3.0/ext/new_allocator.h:136 (libmagma.so+0x0000000acab1)
|
#5 void std::allocator_traits<std::allocator<magma::ThreadPool> >::construct<magma::ThreadPool, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long&, unsigned long&>(std::allocator<magma::ThreadPool>&, magma::ThreadPool*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long&, unsigned long&) /usr/local/include/c++/7.3.0/bits/alloc_traits.h:475 (libmagma.so+0x0000000acab1)
|
#6 std::_Sp_counted_ptr_inplace<magma::ThreadPool, std::allocator<magma::ThreadPool>, (__gnu_cxx::_Lock_policy)2>::_Sp_counted_ptr_inplace<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long&, unsigned long&>(std::allocator<magma::ThreadPool>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long&, unsigned long&) /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:526 (libmagma.so+0x0000000acab1)
|
#7 std::__shared_count<(__gnu_cxx::_Lock_policy)2>::__shared_count<magma::ThreadPool, std::allocator<magma::ThreadPool>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long&, unsigned long&>(std::_Sp_make_shared_tag, magma::ThreadPool*, std::allocator<magma::ThreadPool> const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long&, unsigned long&) /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:637 (libmagma.so+0x0000000acab1)
|
#8 std::__shared_ptr<magma::ThreadPool, (__gnu_cxx::_Lock_policy)2>::__shared_ptr<std::allocator<magma::ThreadPool>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long&, unsigned long&>(std::_Sp_make_shared_tag, std::allocator<magma::ThreadPool> const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long&, unsigned long&) /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:1295 (libmagma.so+0x0000000acab1)
|
#9 std::shared_ptr<magma::ThreadPool>::shared_ptr<std::allocator<magma::ThreadPool>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long&, unsigned long&>(std::_Sp_make_shared_tag, std::allocator<magma::ThreadPool> const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long&, unsigned long&) /usr/local/include/c++/7.3.0/bits/shared_ptr.h:344 (libmagma.so+0x0000000acab1)
|
#10 std::shared_ptr<magma::ThreadPool> std::allocate_shared<magma::ThreadPool, std::allocator<magma::ThreadPool>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long&, unsigned long&>(std::allocator<magma::ThreadPool> const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long&, unsigned long&) /usr/local/include/c++/7.3.0/bits/shared_ptr.h:691 (libmagma.so+0x0000000acab1)
|
#11 std::shared_ptr<magma::ThreadPool> std::make_shared<magma::ThreadPool, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long&, unsigned long&>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long&, unsigned long&) /usr/local/include/c++/7.3.0/bits/shared_ptr.h:707 (libmagma.so+0x0000000acab1)
|
#12 magma::ThreadPool::Create(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long, unsigned long) ../magma/util/thread_pool.cc:58 (libmagma.so+0x0000000acab1)
|
#13 magma::Magma::Open() ../magma/magma/magma.cc:108 (libmagma.so+0x0000000d9ef7)
|
#14 MagmaKVStore::MagmaKVStore(MagmaKVStoreConfig&) ../kv_engine/engines/ep/src/magma-kvstore/magma-kvstore.cc:607 (libep.so+0x00000033fe77)
|
#15 std::_MakeUniq<MagmaKVStore>::__single_object std::make_unique<MagmaKVStore, MagmaKVStoreConfig&>(MagmaKVStoreConfig&) /usr/local/include/c++/7.3.0/bits/unique_ptr.h:825 (libep.so+0x00000021a436)
|
#16 KVStoreFactory::create(KVStoreConfig&) ../kv_engine/engines/ep/src/kvstore.cc:191 (libep.so+0x00000021a436)
|
#17 KVShard::KVShard(unsigned short, unsigned short, Configuration&) ../kv_engine/engines/ep/src/kvshard.cc:56 (libep.so+0x000000249b19)
|
#18 std::_MakeUniq<KVShard>::__single_object std::make_unique<KVShard, unsigned long const&, unsigned long&, Configuration&>(unsigned long const&, unsigned long&, Configuration&) /usr/local/include/c++/7.3.0/bits/unique_ptr.h:825 (libep.so+0x0000002b467d)
|
#19 VBucketMap::VBucketMap(Configuration&, KVBucket&) ../kv_engine/engines/ep/src/vbucketmap.cc:30 (libep.so+0x0000002b467d)
|
#20 KVBucket::KVBucket(EventuallyPersistentEngine&) ../kv_engine/engines/ep/src/kv_bucket.cc:287 (libep.so+0x00000022b691)
|
#21 EPBucket::EPBucket(EventuallyPersistentEngine&) ../kv_engine/engines/ep/src/ep_bucket.cc:254 (libep.so+0x0000001621b2)
|
#22 std::_MakeUniq<EPBucket>::__single_object std::make_unique<EPBucket, EventuallyPersistentEngine&>(EventuallyPersistentEngine&) /usr/local/include/c++/7.3.0/bits/unique_ptr.h:825 (libep.so+0x0000001947d6)
|
#23 EventuallyPersistentEngine::makeBucket(Configuration&) ../kv_engine/engines/ep/src/ep_engine.cc:6583 (libep.so+0x0000001947d6)
|
#24 EventuallyPersistentEngine::initialize(char const*) ../kv_engine/engines/ep/src/ep_engine.cc:2131 (libep.so+0x0000001a55b1)
|
#25 MockTestHarness::create_bucket(bool, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) <null> (ep_testsuite_basic+0x000000445e27)
|
#26 MockTestHarness::reload_engine(EngineIface**, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, bool) <null> (ep_testsuite_basic+0x0000004441e7)
|
#27 test_bug7023 ../kv_engine/engines/ep/tests/ep_testsuite_basic.cc:1708 (ep_testsuite_basic+0x000000480126)
|
#28 execute_test ../kv_engine/programs/engine_testapp/engine_testapp.cc:373 (ep_testsuite_basic+0x000000441a3b)
|
#29 main ../kv_engine/programs/engine_testapp/engine_testapp.cc:624 (ep_testsuite_basic+0x000000442eed)
|
Steps to Reproduce
- Build with TSan on Linux:
mkdir build && cd build
cmake -G Ninja -DCMAKE_BUILD_TYPE=DebugOptimized -DCOUCHBASE_KV_COMMIT_VALIDATION=1 null -DBUILD_ENTERPRISE=1 -DCB_THREADSANITIZER=ON ..
ninja kv_engine/install
- Run the failing test:
ctest -R ep_testsuite_basic.value_eviction.magma