segfault in ep.dylib`Checkpoint::queueDirty when running with jemalloc

When trying to compare jemalloc to tcmalloc, I encountered a segfault in ep_engine when running with jemalloc; inside std::unordered_map

While this is running with a different memory allocator, jemalloc is pretty well tried-and-tested and so I believe this is most likely a bug in our code which a different allocator happens to expose.

Backtrace:

bt

• thread #9: tid = 0x291131, 0x000000010404ed09 ep.dylib`std::_1::hash_iterator<std::1::hash_node<std::1::hash_value_type<std::1::basic_string<char, std::1::char_traits<char>, std::1::allocator<char> >, index_entry>, void*>*> std::1::hash_table<std::1::hash_value_type<std::1::basic_string<char, std::1::char_traits<char>, std::1::allocator<char> >, index_entry>, std::1::unordered_map_hasher<std::1::basic_string<char, std::1::char_traits<char>, std::1::allocator<char> >, std::1::hash_value_type<std::1::basic_string<char, std::1::char_traits<char>, std::1::allocator<char> >, index_entry>, std::1::hash<std::1::basic_string<char, std::1::char_traits<char>, std::1::allocator<char> > >, true>, std::1::unordered_map_equal<std::1::basic_string<char, std::1::char_traits<char>, std::1::allocator<char> >, std::1::hash_value_type<std::1::basic_string<char, std::1::char_traits<char>, std::1::allocator<char> >, index_entry>, std::1::equal_to<std::1::basic_string<char, std::1::char_traits<char>, std::1::allocator<char> > >, true>, std::1::allocator<std::1::hash_value_type<std::1::basic_string<char, std::1::char_traits<char>, std::1::allocator<char> >, index_entry> > >::find<std::1::basic_string<char, std::1::char_traits<char>, std::_1::allocator<char> > >(this=0x000000010a9e3798, __k=0x000000010b092170) + 505 at __hash_table:2029, stop reason = EXC_BAD_ACCESS (code=EXC_I386_GPFLT)
• frame #0: 0x000000010404ed09 ep.dylib`std::_1::hash_iterator<std::1::hash_node<std::1::hash_value_type<std::1::basic_string<char, std::1::char_traits<char>, std::1::allocator<char> >, index_entry>, void*>*> std::1::hash_table<std::1::hash_value_type<std::1::basic_string<char, std::1::char_traits<char>, std::1::allocator<char> >, index_entry>, std::1::unordered_map_hasher<std::1::basic_string<char, std::1::char_traits<char>, std::1::allocator<char> >, std::1::hash_value_type<std::1::basic_string<char, std::1::char_traits<char>, std::1::allocator<char> >, index_entry>, std::1::hash<std::1::basic_string<char, std::1::char_traits<char>, std::1::allocator<char> > >, true>, std::1::unordered_map_equal<std::1::basic_string<char, std::1::char_traits<char>, std::1::allocator<char> >, std::1::hash_value_type<std::1::basic_string<char, std::1::char_traits<char>, std::1::allocator<char> >, index_entry>, std::1::equal_to<std::1::basic_string<char, std::1::char_traits<char>, std::1::allocator<char> > >, true>, std::1::allocator<std::1::hash_value_type<std::1::basic_string<char, std::1::char_traits<char>, std::1::allocator<char> >, index_entry> > >::find<std::1::basic_string<char, std::1::char_traits<char>, std::_1::allocator<char> > >(this=0x000000010a9e3798, __k=0x000000010b092170) + 505 at __hash_table:2029
  frame #1: 0x000000010401cea2 ep.dylib`Checkpoint::queueDirty(SingleThreadedRCPtr<Item> const&, CheckpointManager*) [inlined] std::_1::unordered_map<std::1::basic_string<char, std::1::char_traits<char>, std::1::allocator<char> >, index_entry, std::1::hash<std::1::basic_string<char, std::1::char_traits<char>, std::1::allocator<char> > >, std::1::equal_to<std::1::basic_string<char, std::1::char_traits<char>, std::1::allocator<char> > >, std::1::allocator<std::1::pair<std::1::basic_string<char, std::1::char_traits<char>, std::_1::allocator<char> > const, index_entry> > >::find(this=0x000000010a9e3798, __k=0x000000010b092170) + 12 at unordered_map:968
  frame #2: 0x000000010401ce96 ep.dylib`Checkpoint::queueDirty(this=0x000000010a9e3740, qi=0x000000010387c1d0, checkpointManager=0x0000000109baa798) + 182 at checkpoint.cc:92
  frame #3: 0x000000010402e3df ep.dylib`CheckpointManager::queueDirty(this=0x0000000109baa798, vb=0x000000010387c3a8, qi=0x000000010387c1d0, genSeqno=true) + 1215 at checkpoint.cc:912
  frame #4: 0x000000010405b76c ep.dylib`EventuallyPersistentStore::queueDirty(this=0x0000000103c49300, vb=0x000000010387c3a8, v=0x000000010a96c0b0, plh=0x000000010387c380, tapBackfill=false, notifyReplicator=true, genBySeqno=true) + 268 at ep.cc:2623
  frame #5: 0x000000010405dab9 ep.dylib`EventuallyPersistentStore::set(this=0x0000000103c49300, itm=0x000000010c42ae60, cookie=0x000000010a80e300, force=false, nru='?') + 1897 at ep.cc:668
  frame #6: 0x000000010409eac1 ep.dylib`EventuallyPersistentEngine::store(this=0x0000000103c8d800, cookie=0x000000010a80e300, itm=0x000000010c42ae60, cas=0x000000010a80e560, operation=OPERATION_CAS, vbucket=377) + 433 at ep_engine.cc:2088
  frame #7: 0x000000010409f261 ep.dylib`EventuallyPersistentEngine::store(this=0x0000000103c8d800, cookie=0x000000010a80e300, itm=0x000000010a847080, cas=0x000000010a80e560, operation=OPERATION_APPEND, vbucket=377) + 2385 at ep_engine.cc:2164
  frame #8: 0x000000010409bc5a ep.dylib`EvpStore(handle=0x0000000103c8d800, cookie=0x000000010a80e300, itm=0x000000010a847080, cas=0x000000010a80e560, operation=OPERATION_APPEND, vbucket=377) + 74 at ep_engine.cc:229
  frame #9: 0x00000001023ca071 bucket_engine.dylib`bucket_store(handle=0x00000001023d5248, cookie=0x000000010a80e300, itm=0x000000010a847080, cas=0x000000010a80e560, operation=OPERATION_APPEND, vbucket=377) + 113 at bucket_engine.c:1958
  frame #10: 0x00000001020bdd52 memcached`complete_update_bin(c=0x000000010a80e300) + 722 at memcached.c:936
  frame #11: 0x00000001020ae0b4 memcached`complete_nread(c=0x000000010a80e300) + 292 at memcached.c:5148
  frame #12: 0x00000001020a9c84 memcached`conn_nread(c=0x000000010a80e300) + 68 at memcached.c:6400
  frame #13: 0x00000001020a6982 memcached`run_event_loop(c=0x000000010a80e300) + 162 at connections.c:86
  frame #14: 0x00000001020ac01d memcached`event_handler(fd=43, which=4, arg=0x000000010a80e300) + 573 at memcached.c:6674
  frame #15: 0x00000001021a8f4c libevent_core-2.0.5.dylib`event_base_loop + 1751
  frame #16: 0x00000001020c407f memcached`worker_libevent(arg=0x0000000102ae6e18) + 95 at thread.c:279
  frame #17: 0x0000000102136d8b libplatform.0.1.0.dylib`platform_thread_wrap(arg=0x0000000102806170) + 107 at cb_pthreads.c:19
  frame #18: 0x00007fff8f184899 libsystem_pthread.dylib`_pthread_body + 138
  frame #19: 0x00007fff8f18472a libsystem_pthread.dylib`_pthread_start + 137

LLDB session
===========

Looking at the unordered_map being traversed, it appears that element [3] is corrupted - note the weird key (" 3 0\n ") and the inaccessible _ptr value (0x2030383032343920):

(lldb) up
frame #1: 0x000000010401cea2 ep.dylib`Checkpoint::queueDirty(SingleThreadedRCPtr<Item> const&, CheckpointManager*) [inlined] std::_1::unordered_map<std::1::basic_string<char, std::1::char_traits<char>, std::1::allocator<char> >, index_entry, std::1::hash<std::1::basic_string<char, std::1::char_traits<char>, std::1::allocator<char> > >, std::1::equal_to<std::1::basic_string<char, std::1::char_traits<char>, std::1::allocator<char> > >, std::1::allocator<std::1::pair<std::1::basic_string<char, std::1::char_traits<char>, std::_1::allocator<char> > const, index_entry> > >::find(this=0x000000010a9e3798, __k=0x000000010b092170) + 12 at unordered_map:968
965

{return __table_.key_eq().key_eq();}

966
967 _LIBCPP_INLINE_VISIBILITY
-> 968 iterator find(const key_type& __k)

{return __table_.find(__k);}
969 _LIBCPP_INLINE_VISIBILITY
970 const_iterator find(const key_type& __k) const {return __table_.find(__k);}

971 _LIBCPP_INLINE_VISIBILITY

(lldb) p *this
(std::_1::unordered_map<std::1::basic_string<char, std::1::char_traits<char>, std::1::allocator<char> >, index_entry, std::1::hash<char, std::1::char_traits<char>, std::1::allocator<char> >, std::1::equal_to<std::1::basic_string<char, std::1::char_traits<char>, std::1::allocator<char> > >, std::1::allocator<std::1::pair<const std::1::basic_string<char, std::1::char_traits<char>, std::_1::allocator<char> >, index_entry> > >) $13 = size=6 {
[0] = {
__cc = {
first = "000000000592"
second = {
position =

{ __ptr_ = 0x0000000108567d00 }
mutation_id = 1517
}
}
__nc = {
first = "000000000592"
second = {
position = { __ptr_ = 0x0000000108567d00 }

mutation_id = 1517
}
}
}
[1] = {
__cc = {
first = "000000003623"
second = {
position =

{ __ptr_ = 0x00000001081a5040 }
mutation_id = 1457
}
}
__nc = {
first = "000000003623"
second = {
position = { __ptr_ = 0x00000001081a5040 }

mutation_id = 1457
}
}
}
[2] = {
__cc = {
first = "checkpoint_start"
second = {
position =

{ __ptr_ = 0x000000010a977200 }
mutation_id = 1455
}
}
__nc = {
first = "checkpoint_start"
second = {
position = { __ptr_ = 0x000000010a977200 }

mutation_id = 1455
}
}
}
[3] = {
__cc = {
first = " 3 0\n "
second = {
position =

{ __ptr_ = 0x2030383032343920 }
mutation_id = 2314885599857877024
}
}
__nc = {
first = " 3 0\n "
second = {
position = { __ptr_ = 0x2030383032343920 }

mutation_id = 2314885599857877024
}
}
}
[4] =

{ __cc = <parent is NULL> __nc = <parent is NULL> }

}

(lldb) x 0x2030383032343920
error: memory read failed for 0x2030383032343800