Uploaded image for project: 'Couchbase Server'
  1. Couchbase Server
  2. MB-36285

TSan: lock-order inversion on BasicLinkedList::rangeLock

    XMLWordPrintable

Details

    • Untriaged
    • Ubuntu 64-bit
    • Unknown
    • KV Sprint 2020-February

    Description

      44/59 Test #32: ep_testsuite_dcp.ephemeral.comp_passive ...............***Failed  134.58 sec
       
      2019-10-03T00:54:50.072584+00:00 INFO This version of Couchbase is built without allocator hooks for accurate memory tracking
      ...
      Running [0027/0097]: test producer disk backfill buffer limits (couchstore)...==================
      WARNING: ThreadSanitizer: unlock of an unlocked mutex (or by a wrong thread) (pid=12670)
          #0 pthread_mutex_unlock <null> (libtsan.so.0+0x00000003889b)
          #1 __gthread_mutex_unlock /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:778 (engine_testapp+0x0000004608d1)
          #2 std::mutex::unlock() /usr/local/include/c++/7.3.0/bits/std_mutex.h:121 (engine_testapp+0x00000046a037)
          #3 std::unique_lock<std::mutex>::unlock() /usr/local/include/c++/7.3.0/bits/std_mutex.h:323 (engine_testapp+0x000000478e32)
          #4 BasicLinkedList::RangeIteratorLL::incrOperatorHelper() /mnt/kvengine/kv_engine/engines/ep/src/linked_list.cc:586 (ep.so+0x000000449de3)
          #5 BasicLinkedList::RangeIteratorLL::operator++() /mnt/kvengine/kv_engine/engines/ep/src/linked_list.cc:555 (ep.so+0x000000449a9a)
          #6 SequenceList::RangeIterator::operator++() /mnt/kvengine/kv_engine/engines/ep/src/seqlist.cc:31 (ep.so+0x000000451ca4)
          #7 DCPBackfillMemoryBuffered::scan() /mnt/kvengine/kv_engine/engines/ep/src/dcp/backfill_memory.cc:269 (ep.so+0x0000001fc004)
          #8 DCPBackfillMemoryBuffered::run() /mnt/kvengine/kv_engine/engines/ep/src/dcp/backfill_memory.cc:79 (ep.so+0x0000001fb10a)
          #9 BackfillManager::backfill() /mnt/kvengine/kv_engine/engines/ep/src/dcp/backfill-manager.cc:314 (ep.so+0x0000001ed501)
          #10 BackfillManagerTask::run() /mnt/kvengine/kv_engine/engines/ep/src/dcp/backfill-manager.cc:75 (ep.so+0x0000001eba5d)
          #11 ExecutorThread::run() /mnt/kvengine/kv_engine/engines/ep/src/executorthread.cc:153 (ep.so+0x0000003a5a17)
          #12 launch_executor_thread /mnt/kvengine/kv_engine/engines/ep/src/executorthread.cc:34 (ep.so+0x0000003a4f22)
          #13 CouchbaseThread::run() <null> (libplatform_so.so.0.1.0+0x0000000178ca)
          #14 platform_thread_wrap /mnt/kvengine/platform/src/cb_pthreads.cc:71 (libplatform_so.so.0.1.0+0x000000016aa9)
          #15 <null> <null> (libtsan.so.0+0x000000024feb)
       
        Location is heap block of size 312 at 0x7b4400001180 allocated by main thread:
          #0 malloc <null> (libtsan.so.0+0x0000000274e1)
          #1 cb_malloc /mnt/kvengine/platform/src/cb_malloc.cc:50 (libplatform_so.so.0.1.0+0x00000001c84a)
          #2 operator new(unsigned long) /mnt/kvengine/platform/src/global_new_replacement.cc:71 (engine_testapp+0x0000004cd57b)
          #3 std::_MakeUniq<BasicLinkedList>::__single_object std::make_unique<BasicLinkedList, Vbid&, EPStats&>(Vbid&, EPStats&) usr/local/include/c++/7.3.0/bits/unique_ptr.h:825 (ep.so+0x000000383de5)
          #4 EphemeralVBucket::EphemeralVBucket(Vbid, vbucket_state_t, EPStats&, CheckpointConfig&, KVShard*, long, unsigned long, unsigned long, std::unique_ptr<FailoverTable, std::default_delete<FailoverTable> >, std::unique_ptr<Callback<Vbid const, VBNotifyCtx const&>, std::default_delete<Callback<Vbid const, VBNotifyCtx const&> > >, std::function<void (Vbid)>, std::function<void (void const*, ENGINE_ERROR_CODE)>, std::function<void (Vbid, long)>, Configuration&, EvictionPolicy, std::unique_ptr<Collections::VB::Manifest, std::default_delete<Collections::VB::Manifest> >, vbucket_state_t, unsigned long, unsigned long, bool, nlohmann::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::adl_serializer> const&) /mnt/kvengine/kv_engine/engines/ep/src/ephemeral_vb.cc:81 (ep.so+0x00000037c519)
          #5 EphemeralBucket::makeVBucket(Vbid, vbucket_state_t, KVShard*, std::unique_ptr<FailoverTable, std::default_delete<FailoverTable> >, std::unique_ptr<Callback<Vbid const, VBNotifyCtx const&>, std::default_delete<Callback<Vbid const, VBNotifyCtx const&> > >, std::unique_ptr<Collections::VB::Manifest, std::default_delete<Collections::VB::Manifest> >, vbucket_state_t, long, unsigned long, unsigned long, unsigned long, unsigned long, long, bool, nlohmann::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::adl_serializer> const&) /mnt/kvengine/kv_engine/engines/ep/src/ephemeral_bucket.cc:221 (ep.so+0x00000036c94c)
          #6 KVBucket::setVBucketState_UNLOCKED(Vbid, vbucket_state_t, nlohmann::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::adl_serializer> const&, TransferVB, bool, std::unique_lock<std::mutex>&, folly::SharedMutexImpl<false, void, std::atomic, false, true>::WriteHolder*) /mnt/kvengine/kv_engine/engines/ep/src/kv_bucket.cc:977 (ep.so+0x0000003ef40a)
          #7 KVBucket::setVBucketState(Vbid, vbucket_state_t, nlohmann::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::adl_serializer> const&, TransferVB, void const*) /mnt/kvengine/kv_engine/engines/ep/src/kv_bucket.cc:829 (ep.so+0x0000003ee82e)
          #8 EventuallyPersistentEngine::setVBucketState(void const*, std::function<bool (void const*, unsigned short, void const*, unsigned char, void const*, unsigned int, unsigned char, cb::mcbp::Status, unsigned long, void const*)> const&, Vbid, vbucket_state_t, nlohmann::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::adl_serializer> const&, TransferVB, unsigned long) /mnt/kvengine/kv_engine/engines/ep/src/ep_engine.cc:6341 (ep.so+0x00000032596a)
          #9 setVBucket /mnt/kvengine/kv_engine/engines/ep/src/ep_engine.cc:913 (ep.so+0x00000030b0a2)
          #10 processUnknownCommand /mnt/kvengine/kv_engine/engines/ep/src/ep_engine.cc:1124 (ep.so+0x00000030be96)
          #11 EventuallyPersistentEngine::unknown_command(void const*, cb::mcbp::Request const&, std::function<bool (void const*, unsigned short, void const*, unsigned char, void const*, unsigned int, unsigned char, cb::mcbp::Status, unsigned long, void const*)> const&) /mnt/kvengine/kv_engine/engines/ep/src/ep_engine.cc:1216 (ep.so+0x00000030c5b0)
          #12 ENGINE_ERROR_CODE std::__invoke_impl<ENGINE_ERROR_CODE, ENGINE_ERROR_CODE (EngineIface::*&)(void const*, cb::mcbp::Request const&, std::function<bool (void const*, unsigned short, void const*, unsigned char, void const*, unsigned int, unsigned char, cb::mcbp::Status, unsigned long, void const*)> const&), EngineIface*&, void const*&, cb::mcbp::Request const&, std::function<bool (void const*, unsigned short, void const*, unsigned char, void const*, unsigned int, unsigned char, cb::mcbp::Status, unsigned long, void const*)>&>(std::__invoke_memfun_deref, ENGINE_ERROR_CODE (EngineIface::*&)(void const*, cb::mcbp::Request const&, std::function<bool (void const*, unsigned short, void const*, unsigned char, void const*, unsigned int, unsigned char, cb::mcbp::Status, unsigned long, void const*)> const&), EngineIface*&, void const*&, cb::mcbp::Request const&, std::function<bool (void const*, unsigned short, void const*, unsigned char, void const*, unsigned int, unsigned char, cb::mcbp::Status, unsigned long, void const*)>&) <null> (engine_testapp+0x00000049374e)
          #13 std::__invoke_result<ENGINE_ERROR_CODE (EngineIface::*&)(void const*, cb::mcbp::Request const&, std::function<bool (void const*, unsigned short, void const*, unsigned char, void const*, unsigned int, unsigned char, cb::mcbp::Status, unsigned long, void const*)> const&), EngineIface*&, void const*&, cb::mcbp::Request const&, std::function<bool (void const*, unsigned short, void const*, unsigned char, void const*, unsigned int, unsigned char, cb::mcbp::Status, unsigned long, void const*)>&>::type std::__invoke<ENGINE_ERROR_CODE (EngineIface::*&)(void const*, cb::mcbp::Request const&, std::function<bool (void const*, unsigned short, void const*, unsigned char, void const*, unsigned int, unsigned char, cb::mcbp::Status, unsigned long, void const*)> const&), EngineIface*&, void const*&, cb::mcbp::Request const&, std::function<bool (void const*, unsigned short, void const*, unsigned char, void const*, unsigned int, unsigned char, cb::mcbp::Status, unsigned long, void const*)>&>(ENGINE_ERROR_CODE (EngineIface::*&)(void const*, cb::mcbp::Request const&, std::function<bool (void const*, unsigned short, void const*, unsigned char, void const*, unsigned int, unsigned char, cb::mcbp::Status, unsigned long, void const*)> const&), EngineIface*&, void const*&, cb::mcbp::Request const&, std::function<bool (void const*, unsigned short, void const*, unsigned char, void const*, unsigned int, unsigned char, cb::mcbp::Status, unsigned long, void const*)>&) <null> (engine_testapp+0x00000048e479)
          #14 ENGINE_ERROR_CODE std::_Bind<ENGINE_ERROR_CODE (EngineIface::*(EngineIface*, void const*, std::reference_wrapper<cb::mcbp::Request const>, std::function<bool (void const*, unsigned short, void const*, unsigned char, void const*, unsigned int, unsigned char, cb::mcbp::Status, unsigned long, void const*)>))(void const*, cb::mcbp::Request const&, std::function<bool (void const*, unsigned short, void const*, unsigned char, void const*, unsigned int, unsigned char, cb::mcbp::Status, unsigned long, void const*)> const&)>::__call<ENGINE_ERROR_CODE, , 0ul, 1ul, 2ul, 3ul>(std::tuple<>&&, std::_Index_tuple<0ul, 1ul, 2ul, 3ul>) <null> (engine_testapp+0x000000488c35)
          #15 ENGINE_ERROR_CODE std::_Bind<ENGINE_ERROR_CODE (EngineIface::*(EngineIface*, void const*, std::reference_wrapper<cb::mcbp::Request const>, std::function<bool (void const*, unsigned short, void const*, unsigned char, void const*, unsigned int, unsigned char, cb::mcbp::Status, unsigned long, void const*)>))(void const*, cb::mcbp::Request const&, std::function<bool (void const*, unsigned short, void const*, unsigned char, void const*, unsigned int, unsigned char, cb::mcbp::Status, unsigned long, void const*)> const&)>::operator()<, ENGINE_ERROR_CODE>() <null> (engine_testapp+0x0000004830eb)
          #16 std::_Function_handler<ENGINE_ERROR_CODE (), std::_Bind<ENGINE_ERROR_CODE (EngineIface::*(EngineIface*, void const*, std::reference_wrapper<cb::mcbp::Request const>, std::function<bool (void const*, unsigned short, void const*, unsigned char, void const*, unsigned int, unsigned char, cb::mcbp::Status, unsigned long, void const*)>))(void const*, cb::mcbp::Request const&, std::function<bool (void const*, unsigned short, void const*, unsigned char, void const*, unsigned int, unsigned char, cb::mcbp::Status, unsigned long, void const*)> const&)> >::_M_invoke(std::_Any_data const&) <null> (engine_testapp+0x00000047cc7b)
          #17 std::function<ENGINE_ERROR_CODE ()>::operator()() const <null> (engine_testapp+0x00000046f044)
          #18 call_engine_and_handle_EWOULDBLOCK /mnt/kvengine/kv_engine/programs/engine_testapp/engine_testapp.cc:396 (engine_testapp+0x000000461350)
          #19 mock_engine::unknown_command(void const*, cb::mcbp::Request const&, std::function<bool (void const*, unsigned short, void const*, unsigned char, void const*, unsigned int, unsigned char, cb::mcbp::Status, unsigned long, void const*)> const&) /mnt/kvengine/kv_engine/programs/engine_testapp/engine_testapp.cc:625 (engine_testapp+0x000000462cf8)
          #20 set_vbucket_state(EngineIface*, Vbid, vbucket_state_t) /mnt/kvengine/kv_engine/engines/ep/tests/ep_test_apis.cc:792 (ep_testsuite_dcp.so+0x000000090b7f)
          #21 test_setup(EngineIface*) /mnt/kvengine/kv_engine/engines/ep/tests/ep_testsuite_common.cc:144 (ep_testsuite_dcp.so+0x000000086f2a)
          #22 execute_test /mnt/kvengine/kv_engine/programs/engine_testapp/engine_testapp.cc:1332 (engine_testapp+0x000000465a16)
          #23 main /mnt/kvengine/kv_engine/programs/engine_testapp/engine_testapp.cc:1609 (engine_testapp+0x0000004669d9)
       
        Mutex M936321304527639008 is already destroyed.
       
      SUMMARY: ThreadSanitizer: unlock of an unlocked mutex (or by a wrong thread) (/mnt/kvengine/install/lib/libtsan.so.0+0x3889b) in pthread_mutex_unlock
      

      Attachments

        Issue Links

          No reviews matched the request. Check your Options in the drop-down menu of this sections header.

          Activity

            People

              owend Daniel Owen
              scott.lashley Scott Lashley
              Votes:
              0 Vote for this issue
              Watchers:
              3 Start watching this issue

              Dates

                Created:
                Updated:
                Resolved:

                Gerrit Reviews

                  There are no open Gerrit changes

                  PagerDuty