Uploaded image for project: 'Couchbase Server'
  1. Couchbase Server
  2. MB-37075

jemalloc: crash dereferencing freed TSD when using boost::thread_specific_ptr on Windows

    XMLWordPrintable

Details

    • Bug
    • Resolution: Incomplete
    • Major
    • None
    • master
    • 3rd-party
    • None
    • Triaged
    • Windows 64-bit
    • Unknown

    Description

      Summary

      On Windows, if both jemalloc and boost::thread_specific_ptr are used, then a crash is seen on thread shutdown when attempting to deallocate the data associated with the thread specific ptr.

      Details

      As found when implementing MB-36996, a crash was seen on Windows when http://review.couchbase.org/#/c/118212/ was applied - which replaces our AtomicQueue with using folly::UMPMCQueue. The crash occurred during thread destruction, inside jemalloc's deallocation functions when freeing some thread-local data structures:

       	jemalloc.dll!00007ffaa54daab3()	Unknown
      		 
       	jemalloc.dll!00007ffaa54df63f()	Unknown
      		 
       	jemalloc.dll!00007ffaa54df6a7()	Unknown
      		 
      >	platform_so.dll!cb_sized_free(void * ptr, unsigned __int64 size) Line 82	C++
      		 
       	ep-engine_ep_unit_tests.exe!std::list<std::pair<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalLifetime * const,std::unordered_set<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *,std::hash<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *>,std::equal_to<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *>,std::allocator<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *> > >,std::allocator<std::pair<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalLifetime * const,std::unordered_set<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *,std::hash<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *>,std::equal_to<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *>,std::allocator<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *> > > > >::clear() Line 1393	C++
      		 
       	[Inline Frame] ep-engine_ep_unit_tests.exe!std::_Hash<std::_Umap_traits<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalLifetime *,std::unordered_set<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *,std::hash<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *>,std::equal_to<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *>,std::allocator<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *> >,std::_Uhash_compare<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalLifetime *,std::hash<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalLifetime *>,std::equal_to<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalLifetime *> >,std::allocator<std::pair<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalLifetime * const,std::unordered_set<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *,std::hash<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *>,std::equal_to<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *>,std::allocator<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *> > > >,0> >::clear() Line 637	C++
      		 
       	[Inline Frame] ep-engine_ep_unit_tests.exe!std::_Hash<std::_Umap_traits<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalLifetime *,std::unordered_set<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *,std::hash<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *>,std::equal_to<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *>,std::allocator<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *> >,std::_Uhash_compare<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalLifetime *,std::hash<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalLifetime *>,std::equal_to<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalLifetime *> >,std::allocator<std::pair<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalLifetime * const,std::unordered_set<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *,std::hash<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *>,std::equal_to<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *>,std::allocator<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *> > > >,0> >::erase(std::_List_const_iterator<std::_List_val<std::_List_simple_types<std::pair<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalLifetime * const,std::unordered_set<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *,std::hash<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *>,std::equal_to<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *>,std::allocator<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *> > > > > > _First, std::_List_const_iterator<std::_List_val<std::_List_simple_types<std::pair<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalLifetime * const,std::unordered_set<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *,std::hash<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *>,std::equal_to<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *>,std::allocator<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *> > > > > >) Line 614	C++
      		 
       	ep-engine_ep_unit_tests.exe!std::_Hash<std::_Umap_traits<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalLifetime *,std::unordered_set<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *,std::hash<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *>,std::equal_to<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *>,std::allocator<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *> >,std::_Uhash_compare<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalLifetime *,std::hash<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalLifetime *>,std::equal_to<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalLifetime *> >,std::allocator<std::pair<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalLifetime * const,std::unordered_set<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *,std::hash<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *>,std::equal_to<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *>,std::allocator<folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalCache *> > > >,0> >::erase(folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalLifetime * const & _Keyval) Line 631	C++
      		 
       	ep-engine_ep_unit_tests.exe!folly::SingletonThreadLocal<folly::hazptr_tc<std::atomic>,void,folly::detail::DefaultMake<folly::hazptr_tc<std::atomic> >,void>::LocalLifetime::~LocalLifetime() Line 122	C++
      		 
       	ep-engine_ep_unit_tests.exe!__dyn_tls_dtor(void * __formal, const unsigned long dwReason, void * __formal) Line 119	C++
      		 
       	ntdll.dll!LdrpCallInitRoutine()	Unknown
      		 
       	ntdll.dll!LdrpCallTlsInitializers()	Unknown
      		 
       	ntdll.dll!LdrShutdownThread()	Unknown
      		 
       	ntdll.dll!RtlExitUserThread()	Unknown
      		 
       	kernel32.dll!BaseThreadInitThunk()	Unknown
      		 
       	ntdll.dll!RtlUserThreadStart()	Unknown

      Rebuilding with Debug build type (which enables jemalloc assertions) showed an assertion failing:

      TODO complete details

      Attachments

        No reviews matched the request. Check your Options in the drop-down menu of this sections header.

        Activity

          People

            drigby Dave Rigby (Inactive)
            drigby Dave Rigby (Inactive)
            Votes:
            0 Vote for this issue
            Watchers:
            0 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved:

              Gerrit Reviews

                There are no open Gerrit changes

                PagerDuty