Memcached crashes seen on replica update + rebalance + collections data load

./testrunner -i /tmp/testexec.44768.ini ,get-cbcollect-info=True -t bucket_collections.collection_bucket_params.BucketParams.test_update_replica,num_items=10000,nodes_init=4,replicas=0,bucket_spec=single_bucket.def_scope_fifty_collections,load_spec=def_drop_collection_with_ops,GROUP=P0

(gdb) bt full

#0  bufferevent_incref_and_lock_ (bufev=bufev@entry=0x0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/bufferevent.c:683

        locking = 0x0

        bufev_private = 0x0

#1  0x00007f9f363ddad2 in bufferevent_trigger (bufev=0x0, iotype=<optimized out>, options=65540) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/bufferevent.c:279

No locals.

#2  0x000000000047d706 in Connection::triggerCallback (this=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:923

No locals.

#3  0x000000000046afe8 in thread_libevent_process (fd=<optimized out>, arg=0x7f9f33051aa8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:309

        c = 0x7f9ef76e2100

        __for_range = @0x7f9f2fc6e0a0: {

          _M_h = {<std::__detail::_Hashtable_base<Connection*, std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, std::__detail::_Select1st, std::equal_to<Connection*>, std::hash<Connection*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Hashtable_traits<false, false, true> >> = {<std::__detail::_Hash_code_base<Connection*, std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, std::__detail::_Select1st, std::hash<Connection*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, false>> = {<std::__detail::_Hashtable_ebo_helper<0, std::__detail::_Select1st, true>> = {<std::__detail::_Select1st> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<1, std::hash<Connection*>, true>> = {<std::hash<Connection*>> = {<std::__hash_base<unsigned long, Connection*>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<2, std::__detail::_Mod_range_hashing, true>> = {<std::__detail::_Mod_range_hashing> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<0, std::equal_to<Connection*>, true>> = {<std::equal_to<Connection*>> = {<std::binary_function<Connection*, Connection*, bool>> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Map_base<Connection*, std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, std::allocator<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > > >, std::__detail::_Select1st, std::equal_to<Connection*>, std::hash<Connection*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true>, true>> = {<No data fields>}, <std::__detail::_Insert<Connection*, std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, std::allocator<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > > >, std::__detail::_Select1st, std::equal_to<Connection*>, std::hash<Connection*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true>, false>> = {<std::__detail::_Insert_base<Connection*, std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, std::allocator<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > > >, std::__detail::_Select1st, std::equal_to<Connection*>, std::hash<Connection*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >> = {<No data fields>}, <No data fields>}, <std::__detail::_Rehash_base<Connection*, std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, std::allocator<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > > >, std::__detail::_Select1st, std::equal_to<Connection*>, std::hash<Connection*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true>, std::integral_constant<bool, true> >> = {<No data fields>}, <std::__detail::_Equality<Connection*, std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, std::allocator<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > > >, std::__detail::_Select1st, std::equal_to<Connection*>, std::hash<Connection*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true>, true>> = {<No data fields>}, <std::__detail::_Hashtable_alloc<std::allocator<std::__detail::_Hash_node<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, false> > >> = {<std::__detail::_Hashtable_ebo_helper<0, std::allocator<std::__detail::_Hash_node<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, false> >, true>> = {<std::allocator<std::__detail::_Hash_node<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, false> >> = {<__gnu_cxx::new_allocator<std::__detail::_Hash_node<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, false> >> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, _M_buckets = 0x7f9f2a426980, _M_bucket_count = 7, _M_before_begin = {_M_nxt = 0x7f9f2a4636c0}, _M_element_count = 3, _M_rehash_policy = {static _S_growth_factor = 2, 

              _M_max_load_factor = 1, _M_next_resize = 7}, _M_single_bucket = 0x0}}

        me = @0x7f9f33051aa8: <error reading variable>

        pending = {

          _M_h = {<std::__detail::_Hashtable_base<Connection*, std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, std::__detail::_Select1st, std::equal_to<Connection*>, std::hash<Connection*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Hashtable_traits<false, false, true> >> = {<std::__detail::_Hash_code_base<Connection*, std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, std::__detail::_Select1st, std::hash<Connection*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, false>> = {<std::__detail::_Hashtable_ebo_helper<0, std::__detail::_Select1st, true>> = {<std::__detail::_Select1st> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<1, std::hash<Connection*>, true>> = {<std::hash<Connection*>> = {<std::__hash_base<unsigned long, Connection*>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<2, std::__detail::_Mod_range_hashing, true>> = {<std::__detail::_Mod_range_hashing> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<0, std::equal_to<Connection*>, true>> = {<std::equal_to<Connection*>> = {<std::binary_function<Connection*, Connection*, bool>> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Map_base<Connection*, std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, std::allocator<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > > >, std::__detail::_Select1st, std::equal_to<Connection*>, std::hash<Connection*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true>, true>> = {<No data fields>}, <std::__detail::_Insert<Connection*, std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, std::allocator<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > > >, std::__detail::_Select1st, std::equal_to<Connection*>, std::hash<Connection*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true>, false>> = {<std::__detail::_Insert_base<Connection*, std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, std::allocator<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > > >, std::__detail::_Select1st, std::equal_to<Connection*>, std::hash<Connection*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >> = {<No data fields>}, <No data fields>}, <std::__detail::_Rehash_base<Connection*, std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, std::allocator<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > > >, std::__detail::_Select1st, std::equal_to<Connection*>, std::hash<Connection*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true>, std::integral_constant<bool, true> >> = {<No data fields>}, <std::__detail::_Equality<Connection*, std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, std::allocator<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > > >, std::__detail::_Select1st, std::equal_to<Connection*>, std::hash<Connection*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true>, true>> = {<No data fields>}, <std::__detail::_Hashtable_alloc<std::allocator<std::__detail::_Hash_node<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, false> > >> = {<std::__detail::_Hashtable_ebo_helper<0, std::allocator<std::__detail::_Hash_node<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, false> >, true>> = {<std::allocator<std::__detail::_Hash_node<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, false> >> = {<__gnu_cxx::new_allocator<std::__detail::_Hash_node<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, false> >> = {<No data fields>}, <N---Type <return> to continue, or q <return> to quit---

o data fields>}, <No data fields>}, <No data fields>}, _M_buckets = 0x7f9f2a426980, _M_bucket_count = 7, _M_before_begin = {_M_nxt = 0x7f9f2a4636c0}, _M_element_count = 3, _M_rehash_policy = {static _S_growth_factor = 2, 

              _M_max_load_factor = 1, _M_next_resize = 7}, _M_single_bucket = 0x0}}

        phosphor_internal_category_enabled_285 = {_M_b = {_M_p = 0x0}, static is_always_lock_free = <error reading variable: No global symbol "std::atomic<std::atomic<phosphor::CategoryStatus> const*>::is_always_lock_free".>}

        phosphor_internal_category_enabled_temp_285 = <optimized out>

        phosphor_internal_tpi_wait_285 = {category = 0x5879b0 "mutex", name = 0x589698 "thread_libevent_process::threadLock.wait", type = phosphor::Complete, argument_names = {_M_elems = {0x587937 "this", 0x59d6d3 ""}}, 

          argument_types = {_M_elems = {phosphor::is_pointer, phosphor::is_none}}}

        phosphor_internal_tpi_held_285 = {category = 0x5879b0 "mutex", name = 0x589668 "thread_libevent_process::threadLock.held", type = phosphor::Complete, argument_names = {_M_elems = {0x59d6d3 "", 0x59d6d3 ""}}, argument_types = {

            _M_elems = {phosphor::is_pointer, phosphor::is_none}}}

        phosphor_internal_guard_285 = {tpiWait = 0x589780 <thread_libevent_process(int, short, void*)::phosphor_internal_tpi_wait_285>, tpiHeld = 0x589740 <thread_libevent_process(int, short, void*)::phosphor_internal_tpi_held_285>, 

          enabled = true, mutex = @0x7f9f33051b80, threshold = {__r = 10000000}, start = {__d = {__r = 1351474080452387}}, lockedAt = {__d = {__r = 1351474080453644}}, releasedAt = {__d = {__r = 0}}}

        notify = {<std::_Vector_base<Connection*, std::allocator<Connection*> >> = {_M_impl = {<std::allocator<Connection*>> = {<__gnu_cxx::new_allocator<Connection*>> = {<No data fields>}, <No data fields>}, _M_start = 0x0, 

              _M_finish = 0x0, _M_end_of_storage = <optimized out>}}, <No data fields>}

#4  0x00007f9f363e6faa in event_persist_closure (ev=<optimized out>, base=0x278e4f0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1629

        evcb_callback = 0x46ab10 <thread_libevent_process(int, short, void*)>

        evcb_fd = 25

        evcb_res = 2

        evcb_arg = 0x7f9f33051aa8

#5  event_process_active_single_queue (base=base@entry=0x278e4f0, max_to_process=max_to_process@entry=2147483647, endtime=endtime@entry=0x0, activeq=<optimized out>)

    at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1688

        ev = <optimized out>

        evcb = <optimized out>

        count = 1

#6  0x00007f9f363e754f in event_process_active (base=0x278e4f0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1789

        activeq = <optimized out>

        i = 0

        c = 0

        tv = {tv_sec = 0, tv_usec = 140321758758751}

        maxcb = 2147483647

        endtime = 0x0

        limit_after_prio = 2147483647

#7  event_base_loop (base=0x278e4f0, flags=flags@entry=0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:2012

        evsel = 0x7f9f365fac20 <epollops>

        tv = {tv_sec = 0, tv_usec = 0}

        tv_p = <optimized out>

        res = <optimized out>

        done = 0

        retval = 0

        __func__ = "event_base_loop"

#8  0x0000000000469179 in worker_libevent (arg=0x7f9f33051aa8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:210

        me = @0x7f9f33051aa8: <error reading variable>

#9  0x00007f9f37316827 in run (this=0x7f9f33e49d70) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58

No locals.

#10 platform_thread_wrap (arg=0x7f9f33e49d70) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71

        context = {_M_t = {

            _M_t = {<std::_Tuple_impl<0, CouchbaseThread*, std::default_delete<CouchbaseThread> >> = {<std::_Tuple_impl<1, std::default_delete<CouchbaseThread> >> = {<std::_Head_base<1, std::default_delete<CouchbaseThread>, true>> = {<std::default_delete<CouchbaseThread>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, CouchbaseThread*, false>> = {_M_head_impl = 0x7f9f33e49d70}, <No data fields>}, <No data fields>}}}

#11 0x00007f9f34953dd5 in start_thread () from /lib64/libpthread.so.0

No symbol table info available.

#12 0x00007f9f3467cead in tdestroy_recurse () from /lib64/libc.so.6

No symbol table info available.

#13 0x0000000000000000 in ?? ()

No symbol table info available.

(gdb) 

(gdb)