[Collections] - memcached crashes seen during collection data load + durability + graceful failover and rebalance out

./testrunner -i /tmp/win10-bucket-ops.ini sdk_client_pool=True,rerun=False,crash_warning=True -t bucket_collections.collections_rebalance.CollectionsRebalance.test_data_load_collections_with_graceful_failover_rebalance_out,nodes_init=5,nodes_failover=1,override_spec_params=durability;replicas,durability=MAJORITY,replicas=Bucket.ReplicaNum.TWO,bucket_spec=multi_bucket.buckets_for_rebalance_tests,data_load_stage=during,GROUP=durability_majority

2020-05-27 01:15:24,082 | test  | INFO    | pool-2-thread-25 | [task:check:318] Rebalance - status: running, progress: 0

2020-05-27 01:15:40,487 | test  | ERROR   | pool-2-thread-25 | [rest_client:_rebalance_status_and_progress:1478] {u'errorMessage': u'Rebalance failed. See logs for detailed reason. You can try again.', u'status': u'none'} - rebalance failed

2020-05-27 01:15:40,562 | test  | INFO    | pool-2-thread-25 | [rest_client:print_UI_logs:2535] Latest logs from UI on 172.23.98.196:

2020-05-27 01:15:40,562 | test  | ERROR   | pool-2-thread-25 | [rest_client:print_UI_logs:2537] {u'code': 0, u'module': u'ns_orchestrator', u'type': u'critical', u'node': u'ns_1@172.23.98.196', u'tstamp': 1590567335524L, u'shortText': u'message', u'serverTime': u'2020-05-27T01:15:35.524Z', u'text': u'Rebalance exited with reason {mover_crashed,\n                              {unexpected_exit,\n                               {\'EXIT\',<0.9674.5>,\n                                {{{{{child_interrupted,\n                                     {\'EXIT\',<17621.5080.0>,socket_closed}},\n                                    [{dcp_replicator,spawn_and_wait,1,\n                                      [{file,"src/dcp_replicator.erl"},\n                                       {line,266}]},\n                                     {dcp_replicator,handle_call,3,\n                                      [{file,"src/dcp_replicator.erl"},\n                                       {line,127}]},\n                                     {gen_server,try_handle_call,4,\n                                      [{file,"gen_server.erl"},{line,636}]},\n                                     {gen_server,handle_msg,6,\n                                      [{file,"gen_server.erl"},{line,665}]},\n                                     {proc_lib,init_p_do_apply,3,\n                                      [{file,"proc_lib.erl"},{line,247}]}]},\n                                   {gen_server,call,\n                                    [<17621.5078.0>,{takeover,471},infinity]}},\n                                  {gen_server,call,\n                                   [\'replication_manager-default\',\n                                    {dcp_takeover,\'ns_1@172.23.121.10\',471},\n                                    infinity]}},\n                                 {gen_server,call,\n                                  [{\'janitor_agent-default\',\n                                    \'ns_1@172.23.104.186\'},\n                                   {if_rebalance,<0.5701.5>,\n                                    {dcp_takeover,\'ns_1@172.23.121.10\',471}},\n                                   infinity]}}}}}.\nRebalance Operation Id = 7cc85f30de3029fd3a9a829e73fcf3c6'}

(gdb) bt full

#0  bufferevent_incref_and_lock_ (bufev=bufev@entry=0x0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/bufferevent.c:683

        locking = 0x0

        bufev_private = 0x0

#1  0x00007f932169ead2 in bufferevent_trigger (bufev=0x0, iotype=<optimized out>, options=65540) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/bufferevent.c:279

No locals.

#2  0x000000000047d836 in Connection::triggerCallback (this=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:923

No locals.

#3  0x000000000046b118 in thread_libevent_process (fd=<optimized out>, arg=0x7f931e450aa8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:309

        c = 0x7f931e3c1400

        __for_range = @0x7f9316ff30a0: {

          _M_h = {<std::__detail::_Hashtable_base<Connection*, std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, std::__detail::_Select1st, std::equal_to<Connection*>, std::hash<Connection*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Hashtable_traits<false, false, true> >> = {<std::__detail::_Hash_code_base<Connection*, std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, std::__detail::_Select1st, std::hash<Connection*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, false>> = {<std::__detail::_Hashtable_ebo_helper<0, std::__detail::_Select1st, true>> = {<std::__detail::_Select1st> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<1, std::hash<Connection*>, true>> = {<std::hash<Connection*>> = {<std::__hash_base<unsigned long, Connection*>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<2, std::__detail::_Mod_range_hashing, true>> = {<std::__detail::_Mod_range_hashing> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<0, std::equal_to<Connection*>, true>> = {<std::equal_to<Connection*>> = {<std::binary_function<Connection*, Connection*, bool>> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Map_base<Connection*, std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, std::allocator<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > > >, std::__detail::_Select1st, std::equal_to<Connection*>, std::hash<Connection*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true>, true>> = {<No data fields>}, <std::__detail::_Insert<Connection*, std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, std::allocator<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > > >, std::__detail::_Select1st, std::equal_to<Connection*>, std::hash<Connection*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true>, false>> = {<std::__detail::_Insert_base<Connection*, std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, std::allocator<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > > >, std::__detail::_Select1st, std::equal_to<Connection*>, std::hash<Connection*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >> = {<No data fields>}, <No data fields>}, <std::__detail::_Rehash_base<Connection*, std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, std::allocator<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > > >, std::__detail::_Select1st, std::equal_to<Connection*>, std::hash<Connection*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true>, std::integral_constant<bool, true> >> = {<No data fields>}, <std::__detail::_Equality<Connection*, std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, std::allocator<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > > >, std::__detail::_Select1st, std::equal_to<Connection*>, std::hash<Connection*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true>, true>> = {<No data fields>}, <std::__detail::_Hashtable_alloc<std::allocator<std::__detail::_Hash_node<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, false> > >> = {<std::__detail::_Hashtable_ebo_helper<0, std::allocator<std::__detail::_Hash_node<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, false> >, true>> = {<std::allocator<std::__detail::_Hash_node<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, false> >> = {<__gnu_cxx::new_allocator<std::__detail::_Hash_node<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, false> >> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, _M_buckets = 0x7f92bfb3e320, _M_bucket_count = 3, _M_before_begin = {_M_nxt = 0x7f92bfbb5de0}, _M_element_count = 1, _M_rehash_policy = {static _S_growth_factor = 2, 

              _M_max_load_factor = 1, _M_next_resize = 3}, _M_single_bucket = 0x0}}

        me = @0x7f931e450aa8: <error reading variable>

        pending = {

          _M_h = {<std::__detail::_Hashtable_base<Connection*, std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, std::__detail::_Select1st, std::equal_to<Connection*>, std::hash<Connection*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Hashtable_traits<false, false, true> >> = {<std::__detail::_Hash_code_base<Connection*, std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, std::__detail::_Select1st, std::hash<Connection*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, false>> = {<std::__detail::_Hashtable_ebo_helper<0, std::__detail::_Select1st, true>> = {<std::__detail::_Select1st> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<1, std::hash<Connection*>, true>> = {<std::hash<Connection*>> = {<std::__hash_base<unsigned long, Connection*>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<2, std::__detail::_Mod_range_hashing, true>> = {<std::__detail::_Mod_range_hashing> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<0, std::equal_to<Connection*>, true>> = {<std::equal_to<Connection*>> = {<std::binary_function<Connection*, Connection*, bool>> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Map_base<Connection*, std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, std::allocator<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > > >, std::__detail::_Select1st, std::equal_to<Connection*>, std::hash<Connection*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true>, true>> = {<No data fields>}, <std::__detail::_Insert<Connection*, std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, std::allocator<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > > >, std::__detail::_Select1st, std::equal_to<Connection*>, std::hash<Connection*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true>, false>> = {<std::__detail::_Insert_base<Connection*, std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, std::allocator<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > > >, std::__detail::_Select1st, std::equal_to<Connection*>, std::hash<Connection*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >> = {<No data fields>}, <No data fields>}, <std::__detail::_Rehash_base<Connection*, std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, std::allocator<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > > >, std::__detail::_Select1st, std::equal_to<Connection*>, std::hash<Connection*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true>, std::integral_constant<bool, true> >> = {<No data fields>}, <std::__detail::_Equality<Connection*, std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, std::allocator<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > > >, std::__detail::_Select1st, std::equal_to<Connection*>, std::hash<Connection*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true>, true>> = {<No data fields>}, <std::__detail::_Hashtable_alloc<std::allocator<std::__detail::_Hash_node<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, false> > >> = {<std::__detail::_Hashtable_ebo_helper<0, std::allocator<std::__detail::_Hash_node<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, false> >, true>> = {<std::allocator<std::__detail::_Hash_node<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, false> >> = {<__gnu_cxx::new_allocator<std::__detail::_Hash_node<std::pair<Connection* const, std::vector<std::pair<Cookie*, ENGINE_ERROR_CODE>, std::allocator<std::pair<Cookie*, ENGINE_ERROR_CODE> > > >, false> >> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, _M_buckets = 0x7f92bfb3e320, _M_bucket_count = 3, _M_before_begin = {_M_nxt = 0x7f92bfbb5de0}, _M_element_count = 1, _M_rehash_policy = {static _S_growth_factor = 2, 

              _M_max_load_factor = 1, _M_next_resize = 3}, _M_single_bucket = 0x0}}

        phosphor_internal_category_enabled_285 = {_M_b = {_M_p = 0x0}, static is_always_lock_free = <error reading variable: No global symbol "std::atomic<std::atomic<phosphor::CategoryStatus> const*>::is_always_lock_free".>}

        phosphor_internal_category_enabled_temp_285 = <optimized out>

        phosphor_internal_tpi_wait_285 = {category = 0x587cf0 "mutex", name = 0x5899d8 "thread_libevent_process::threadLock.wait", type = phosphor::Complete, argument_names = {_M_elems = {0x587c77 "this", 0x59da13 ""}}, 

          argument_types = {_M_elems = {phosphor::is_pointer, phosphor::is_none}}}

        phosphor_internal_tpi_held_285 = {category = 0x587cf0 "mutex", name = 0x5899a8 "thread_libevent_process::threadLock.held", type = phosphor::Complete, argument_names = {_M_elems = {0x59da13 "", 0x59da13 ""}}, argument_types = {

            _M_elems = {phosphor::is_pointer, phosphor::is_none}}}

        phosphor_internal_guard_285 = {tpiWait = 0x589ac0 <thread_libevent_process(int, short, void*)::phosphor_internal_tpi_wait_285>, tpiHeld = 0x589a80 <thread_libevent_process(int, short, void*)::phosphor_internal_tpi_held_285>, 

          enabled = true, mutex = @0x7f931e450b80, threshold = {__r = 10000000}, start = {__d = {__r = 1989683242172862}}, lockedAt = {__d = {__r = 1989683242173848}}, releasedAt = {__d = {__r = 0}}}

        notify = {<std::_Vector_base<Connection*, std::allocator<Connection*> >> = {_M_impl = {<std::allocator<Connection*>> = {<__gnu_cxx::new_allocator<Connection*>> = {<No data fields>}, <No data fields>}, _M_start = 0x0, 

              _M_finish = 0x0, _M_end_of_storage = <optimized out>}}, <No data fields>}

#4  0x00007f93216a7faa in event_persist_closure (ev=<optimized out>, base=0x12f94f0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1629

---Type <return> to continue, or q <return> to quit---

        evcb_callback = 0x46ac40 <thread_libevent_process(int, short, void*)>

        evcb_fd = 25

        evcb_res = 2

        evcb_arg = 0x7f931e450aa8

#5  event_process_active_single_queue (base=base@entry=0x12f94f0, max_to_process=max_to_process@entry=2147483647, endtime=endtime@entry=0x0, activeq=<optimized out>)

    at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1688

        ev = <optimized out>

        evcb = <optimized out>

        count = 1

#6  0x00007f93216a854f in event_process_active (base=0x12f94f0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1789

        activeq = <optimized out>

        i = 0

        c = 0

        tv = {tv_sec = 0, tv_usec = 140269869717231}

        maxcb = 2147483647

        endtime = 0x0

        limit_after_prio = 2147483647

#7  event_base_loop (base=0x12f94f0, flags=flags@entry=0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:2012

        evsel = 0x7f93218bbc20 <epollops>

        tv = {tv_sec = 0, tv_usec = 0}

        tv_p = <optimized out>

        res = <optimized out>

        done = 0

        retval = 0

        __func__ = "event_base_loop"

#8  0x00000000004692a9 in worker_libevent (arg=0x7f931e450aa8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:210

        me = @0x7f931e450aa8: <error reading variable>

#9  0x00007f93225d7777 in run (this=0x7f931f24dd70) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58

No locals.

#10 platform_thread_wrap (arg=0x7f931f24dd70) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71

        context = {_M_t = {

            _M_t = {<std::_Tuple_impl<0, CouchbaseThread*, std::default_delete<CouchbaseThread> >> = {<std::_Tuple_impl<1, std::default_delete<CouchbaseThread> >> = {<std::_Head_base<1, std::default_delete<CouchbaseThread>, true>> = {<std::default_delete<CouchbaseThread>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, CouchbaseThread*, false>> = {_M_head_impl = 0x7f931f24dd70}, <No data fields>}, <No data fields>}}}

#11 0x00007f931fc14e25 in start_thread () from /lib64/libpthread.so.0

No symbol table info available.

#12 0x00007f931f94234d in clone () from /lib64/libc.so.6

No symbol table info available.

(gdb) 

(gdb) bt full

#0  0x00007fe28351f5d7 in raise () from /lib64/libc.so.6

No symbol table info available.

#1  0x00007fe283520cc8 in abort () from /lib64/libc.so.6

No symbol table info available.

#2  0x00007fe284071195 in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/gcc-7.3.0/libstdc++-v3/libsupc++/vterminate.cc:95

        terminating = false

        t = <optimized out>

#3  0x000000000054a9f2 in backtrace_terminate_handler () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:86

No locals.

#4  0x00007fe28406ef86 in __cxxabiv1::__terminate (handler=<optimized out>) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/libsupc++/eh_terminate.cc:47

No locals.

#5  0x00007fe28406efd1 in std::terminate () at /tmp/deploy/gcc-7.3.0/libstdc++-v3/libsupc++/eh_terminate.cc:57

No locals.

#6  0x00007fe28406f213 in __cxxabiv1::__cxa_throw (obj=obj@entry=0x7fe23005d250, tinfo=0x7fe284354ad8 <typeinfo for std::underflow_error>, dest=0x7fe2840844f0 <std::underflow_error::~underflow_error()>)

    at /tmp/deploy/gcc-7.3.0/libstdc++-v3/libsupc++/eh_throw.cc:93

        globals = <optimized out>

        header = 0x7fe23005d1d0

#7  0x00007fe28777bee4 in cb::ThrowExceptionUnderflowPolicy<unsigned long>::underflow (this=<optimized out>, desired=@0x7fe261775008: 18446744073709551564, current=<optimized out>, arg=<optimized out>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/include/platform/non_negative_counter.h:48

No locals.

#8  0x00007fe28781a158 in store (desired=<optimized out>, this=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/include/platform/non_negative_counter.h:106

No locals.

#9  operator= (val=<optimized out>, this=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/include/platform/non_negative_counter.h:226

No locals.

#10 EPVBucket::setNumTotalItems (this=<optimized out>, totalItems=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_vb.cc:379

No locals.

#11 0x00007fe2878ea8a1 in Warmup::estimateDatabaseItemCount (this=0x7fe278347000, shardId=3) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/warmup.cc:1206

        vbItemCount = 18446744073709551564

        vbState = <optimized out>

        itr = {<folly::detail::IteratorFacade<folly::AtomicHashMap<unsigned short, std::shared_ptr<VBucket> >::ahm_iterator<folly::AtomicHashMap<unsigned short, std::shared_ptr<VBucket>, std::hash<unsigned short>, std::equal_to<unsigned short>, std::allocator<char>, folly::AtomicHashArrayLinearProbeFcn, folly::Identity>, std::pair<unsigned short const, std::shared_ptr<VBucket> >, folly::AtomicHashArray<unsigned short, std::shared_ptr<VBucket>, std::hash<unsigned short>, std::equal_to<unsigned short>, std::allocator<char>, folly::AtomicHashArrayLinearProbeFcn, folly::Identity>::aha_iterator<folly::AtomicHashArray<unsigned short, std::shared_ptr<VBucket>, std::hash<unsigned short>, std::equal_to<unsigned short>, std::allocator<char>, folly::AtomicHashArrayLinearProbeFcn, folly::Identity>, std::pair<unsigned short const, std::shared_ptr<VBucket> > > >, std::pair<unsigned short const, std::shared_ptr<VBucket> >, std::forward_iterator_tag>> = {<No data fields>}, ahm_ = 0x7fe278347190, subMap_ = 0, 

          subIt_ = {<folly::detail::IteratorFacade<folly::AtomicHashArray<unsigned short, std::shared_ptr<VBucket>, std::hash<unsigned short>, std::equal_to<unsigned short>, std::allocator<char>, folly::AtomicHashArrayLinearProbeFcn, folly::Identity>::aha_iterator<folly::AtomicHashArray<unsigned short, std::shared_ptr<VBucket>, std::hash<unsigned short>, std::equal_to<unsigned short>, std::allocator<char>, folly::AtomicHashArrayLinearProbeFcn, folly::Identity>, std::pair<unsigned short const, std::shared_ptr<VBucket> > >, std::pair<unsigned short const, std::shared_ptr<VBucket> >, std::forward_iterator_tag>> = {<No data fields>}, aha_ = 0x7fe263f1c000, offset_ = 147}}

        vbid = {vbid = 147}

        item_count = 73030

#12 0x00007fe2878f66a0 in WarmupEstimateDatabaseItemCount::run (this=0x7fe24434c1f0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/warmup.cc:252

        phosphor_internal_category_enabled_251 = {_M_b = {_M_p = 0x0}, static is_always_lock_free = <error reading variable: No global symbol "std::atomic<std::atomic<phosphor::CategoryStatus> const*>::is_always_lock_free".>}

        phosphor_internal_category_enabled_temp_251 = <optimized out>

        phosphor_internal_tpi_251 = {category = 0x292b73 <Address 0x292b73 out of bounds>, name = 0x2b3a98 <Address 0x2b3a98 out of bounds>, type = phosphor::Complete, argument_names = {_M_elems = {

              0x2bf67b <Address 0x2bf67b out of bounds>, 0x2bf67b <Address 0x2bf67b out of bounds>}}, argument_types = {_M_elems = {phosphor::is_none, phosphor::is_none}}}

        phosphor_internal_guard_251 = {tpi = 0x7fe287c75fa0 <WarmupEstimateDatabaseItemCount::run()::phosphor_internal_tpi_251>, enabled = true, arg1 = {<No data fields>}, arg2 = {<No data fields>}, start = {__d = {

              __r = 1990429643227166}}}

#13 0x00007fe287869b53 in GlobalTask::execute (this=0x7fe24434c1f0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/globaltask.cc:73

        guard = {previous = 0x0}

#14 0x00007fe2878623ff in ExecutorThread::run (this=0x7fe2821cafc0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/executorthread.cc:188

        curTaskDescr = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, 

            _M_p = 0x7fe2820ab5b0 <Address 0x7fe2820ab5b0 out of bounds>}, _M_string_length = 37, {_M_local_buf = "%\000\000\000\000\000\000\000m[\000\000\000\000\000", _M_allocated_capacity = 37}}

        woketime = <optimized out>

        scheduleOverhead = <optimized out>

        again = <optimized out>

        runtime = <optimized out>

        q = <optimized out>

        tick = 5 '\005'

        guard = {engine = 0x0}

#15 0x00007fe286275777 in run (this=0x7fe2820aa2f0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58

No locals.

#16 platform_thread_wrap (arg=0x7fe2820aa2f0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71

        context = {_M_t = {

            _M_t = {<std::_Tuple_impl<0, CouchbaseThread*, std::default_delete<CouchbaseThread> >> = {<std::_Tuple_impl<1, std::default_delete<CouchbaseThread> >> = {<std::_Head_base<1, std::default_delete<CouchbaseThread>, true>> = {<std::default_delete<CouchbaseThread>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, CouchbaseThread*, false>> = {_M_head_impl = 0x7fe2820aa2f0}, <No data fields>}, <No data fields>}}}

#17 0x00007fe2838b2df5 in start_thread () from /lib64/libpthread.so.0

No symbol table info available.

#18 0x00007fe2835e01ad in clone () from /lib64/libc.so.6

No symbol table info available.

(gdb)