Details
-
Bug
-
Resolution: Fixed
-
Critical
-
5.5.0
-
Untriaged
-
Centos 64-bit
-
-
Yes
Description
Build 5.5.0-2617.
Setup:
- 5 nodes
- 1 bucket
- 1B items (~1KB size)
- 15K ops/sec (read-heavy)
172.23.96.104 was removed from the cluster (i.e., rebalance out).
From ns_server log:
[user:error,2018-04-30T19:09:20.703-07:00,ns_1@172.23.96.100:<0.2256.0>:ns_orchestrator:do_log_rebalance_completion:1122]Rebalance exited with reason {mover_crashed,
|
{unexpected_exit,
|
{'EXIT',<0.20158.15>,
|
{wait_seqno_persisted_failed,"bucket-1",405,
|
976932,
|
[{'ns_1@172.23.96.103',
|
{'EXIT',
|
{{{badmatch,{error,closed}},
|
[{dcp_proxy,handle_packet,2,
|
[{file,"src/dcp_proxy.erl"},
|
{line,172}]},
|
{dcp_proxy,process_data_loop,3,
|
[{file,"src/dcp_proxy.erl"},
|
{line,334}]},
|
{dcp_proxy,handle_info,2,
|
[{file,"src/dcp_proxy.erl"},
|
{line,100}]},
|
{gen_server,handle_msg,5,
|
[{file,"gen_server.erl"},{line,604}]},
|
{proc_lib,init_p_do_apply,3,
|
[{file,"proc_lib.erl"},{line,239}]}]},
|
{gen_server,call,
|
[{'janitor_agent-bucket-1',
|
'ns_1@172.23.96.103'},
|
{if_rebalance,<0.18046.15>,
|
{wait_seqno_persisted,405,976932}},
|
infinity]}}}}]}}}}
|
From memcached log:
2018-04-30T19:09:16.901313Z CRITICAL Breakpad caught a crash (Couchbase version 5.5.0-2617). Writing crash dump to /opt/couchbase/var/lib/couchbase/crash/51ef6b9a-c567-03e5-537a0308-52a702d4.dmp before terminati
|
ng.
|
2018-04-30T19:09:16.901323Z CRITICAL Stack backtrace of crashed thread:
|
2018-04-30T19:09:16.924649Z CRITICAL /opt/couchbase/bin/../lib/libmcd_util.so.1.0.0() [0x7fb4049c5000+0xfe50]
|
2018-04-30T19:09:16.924663Z CRITICAL /opt/couchbase/bin/../lib/libmcd_util.so.1.0.0(_ZN15google_breakpad16ExceptionHandler12GenerateDumpEPNS0_12CrashContextE+0x3ce) [0x7fb4049c5000+0x231ae]
|
2018-04-30T19:09:16.924666Z CRITICAL /opt/couchbase/bin/../lib/libmcd_util.so.1.0.0(_ZN15google_breakpad16ExceptionHandler13SignalHandlerEiP9siginfo_tPv+0x94) [0x7fb4049c5000+0x234c4]
|
2018-04-30T19:09:16.924673Z CRITICAL /lib64/libpthread.so.0() [0x7fb40167e000+0xf370]
|
2018-04-30T19:09:16.924697Z CRITICAL /lib64/libc.so.6(gsignal+0x37) [0x7fb4012bd000+0x351d7]
|
2018-04-30T19:09:16.924714Z CRITICAL /lib64/libc.so.6(abort+0x148) [0x7fb4012bd000+0x368c8]
|
2018-04-30T19:09:16.924735Z CRITICAL /opt/couchbase/bin/../lib/libstdc++.so.6(_ZN9__gnu_cxx27__verbose_terminate_handlerEv+0x125) [0x7fb401db3000+0x91195]
|
2018-04-30T19:09:16.924740Z CRITICAL /opt/couchbase/bin/../lib/libmcd_util.so.1.0.0() [0x7fb4049c5000+0x21abb]
|
2018-04-30T19:09:16.924755Z CRITICAL /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7fb401db3000+0x8ef86]
|
2018-04-30T19:09:16.924768Z CRITICAL /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7fb401db3000+0x8efd1]
|
2018-04-30T19:09:16.924781Z CRITICAL /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7fb401db3000+0x8f213]
|
2018-04-30T19:09:16.924792Z CRITICAL /opt/couchbase/bin/../lib/../lib/ep.so() [0x7fb3ecfcb000+0x82eeb]
|
2018-04-30T19:09:16.924796Z CRITICAL /opt/couchbase/bin/../lib/../lib/ep.so() [0x7fb3ecfcb000+0x7a6ef]
|
2018-04-30T19:09:16.924800Z CRITICAL /opt/couchbase/bin/../lib/../lib/ep.so() [0x7fb3ecfcb000+0x8033d]
|
2018-04-30T19:09:16.924802Z CRITICAL /opt/couchbase/bin/../lib/../lib/ep.so() [0x7fb3ecfcb000+0x80579]
|
2018-04-30T19:09:16.924805Z CRITICAL /opt/couchbase/bin/../lib/../lib/ep.so() [0x7fb3ecfcb000+0x80869]
|
2018-04-30T19:09:16.924809Z CRITICAL /opt/couchbase/bin/../lib/../lib/ep.so() [0x7fb3ecfcb000+0xc2449]
|
2018-04-30T19:09:16.924813Z CRITICAL /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0() [0x7fb403522000+0x89a7]
|
2018-04-30T19:09:16.924817Z CRITICAL /lib64/libpthread.so.0() [0x7fb40167e000+0x7dc5]
|
2018-04-30T19:09:16.924850Z CRITICAL /lib64/libc.so.6(clone+0x6d) [0x7fb4012bd000+0xf776d]
|
One more dump a few seconds later:
2018-04-30T19:09:24.944842Z CRITICAL Breakpad caught a crash (Couchbase version 5.5.0-2617). Writing crash dump to /opt/couchbase/var/lib/couchbase/crash/2e47702f-3d3c-e182-4e44ec65-74fcde19.dmp before terminati
|
ng.
|
2018-04-30T19:09:24.944863Z CRITICAL Stack backtrace of crashed thread:
|
2018-04-30T19:09:24.945017Z CRITICAL /opt/couchbase/bin/../lib/libmcd_util.so.1.0.0() [0x7fbc6ee28000+0xfe50]
|
2018-04-30T19:09:24.945030Z CRITICAL /opt/couchbase/bin/../lib/libmcd_util.so.1.0.0(_ZN15google_breakpad16ExceptionHandler12GenerateDumpEPNS0_12CrashContextE+0x3ce) [0x7fbc6ee28000+0x231ae]
|
2018-04-30T19:09:24.945038Z CRITICAL /opt/couchbase/bin/../lib/libmcd_util.so.1.0.0(_ZN15google_breakpad16ExceptionHandler13SignalHandlerEiP9siginfo_tPv+0x94) [0x7fbc6ee28000+0x234c4]
|
2018-04-30T19:09:24.945048Z CRITICAL /lib64/libpthread.so.0() [0x7fbc6bae1000+0xf370]
|
2018-04-30T19:09:24.945057Z CRITICAL /opt/couchbase/bin/../lib/../lib/ep.so() [0x7fbc573cb000+0x139fe5]
|
2018-04-30T19:09:24.945061Z CRITICAL /opt/couchbase/bin/../lib/../lib/ep.so() [0x7fbc573cb000+0x1091ce]
|
2018-04-30T19:09:24.945065Z CRITICAL /opt/couchbase/bin/../lib/../lib/ep.so() [0x7fbc573cb000+0x13eac4]
|
2018-04-30T19:09:24.945068Z CRITICAL /opt/couchbase/bin/../lib/../lib/ep.so() [0x7fbc573cb000+0x13f0c0]
|
2018-04-30T19:09:24.945071Z CRITICAL /opt/couchbase/bin/../lib/../lib/ep.so() [0x7fbc573cb000+0x1424c7]
|
2018-04-30T19:09:24.945076Z CRITICAL /opt/couchbase/bin/../lib/../lib/ep.so() [0x7fbc573cb000+0xc2449]
|
2018-04-30T19:09:24.945082Z CRITICAL /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0() [0x7fbc6d985000+0x89a7]
|
2018-04-30T19:09:24.945087Z CRITICAL /lib64/libpthread.so.0() [0x7fbc6bae1000+0x7dc5]
|
2018-04-30T19:09:24.945116Z CRITICAL /lib64/libc.so.6(clone+0x6d) [0x7fbc6b720000+0xf776d]
|
The first crash:
(gdb) bt full
|
#0 0x00007fb4012f21d7 in _quicksort (pbase=0x0, total_elems=<optimized out>, size=32, cmp=0x7145f0 <stderr>, arg=0x7fa504000980) at qsort.c:242
|
c = 8 '\b'
|
hi = 0x0
|
lo = 0x6 <Address 0x6 out of bounds>
|
trav = 0x1492 ""
|
end_ptr = 0x0
|
tmp_ptr = 0x16b2 ""
|
thresh = <optimized out>
|
run_ptr = 0x7fb3bcadbd98 <Address 0x7fb3bcadbd98 out of bounds>
|
base_ptr = <optimized out>
|
max_thresh = 0
|
#1 0x00007fb2ad7f1a90 in ?? ()
|
No symbol table info available.
|
#2 0x00007fb2ad7f1a50 in ?? ()
|
No symbol table info available.
|
#3 0x00007fb2a8d974d0 in ?? ()
|
No symbol table info available.
|
#4 0x00007fb2ad7f1a30 in ?? ()
|
No symbol table info available.
|
#5 0x00007fb3ed168298 in ?? () from /opt/couchbase/lib/ep.so
|
No symbol table info available.
|
#6 0x00007fb3ed003184 in __gthread_mutex_unlock (__mutex=<error reading variable: Cannot access memory at address 0xffffffffffffff99>)
|
at /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:778
|
No locals.
|
#7 unlock (this=<error reading variable: Cannot access memory at address 0xffffffffffffff99>) at /usr/local/include/c++/7.3.0/bits/std_mutex.h:121
|
No locals.
|
#8 ~lock_guard (this=<synthetic pointer>, __in_chrg=<optimized out>) at /usr/local/include/c++/7.3.0/bits/std_mutex.h:168
|
No locals.
|
#9 CheckpointManager::getItemsForCursor (this=<error reading variable: Cannot access memory at address 0xffffffffffffff99>, name=<error reading variable: Cannot access memory at address 0xffffffffffffffa9>,
|
items=<error reading variable: Cannot access memory at address 0xffffffffffffffc9>, approxLimit=<error reading variable: Cannot access memory at address 0xffffffffffffffb9>)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/checkpoint.cc:1276
|
lh = <error reading variable lh (Cannot access memory at address 0xffffffffffffff99)>
|
it = {_M_node = 0x7fb2ad7f1a70}
|
cursor = <optimized out>
|
result = <optimized out>
|
itemCount = <optimized out>
|
The second crash:
(gdb) bt full
|
#0 0x00007fbc57504fe5 in VBucketBGFetchItem (meta_only=false, init_time=..., c=0x0, value_=0x0, this=0x7fbc279f7a00)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket_bgfetch_item.h:38
|
No locals.
|
#1 VBucketBGFetchItem (meta_only=false, c=0x0, this=0x7fbc279f7a00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket_bgfetch_item.h:28
|
No locals.
|
#2 make_unique<VBucketBGFetchItem, std::nullptr_t, bool> () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/include/platform/make_unique.h:45
|
No locals.
|
#3 batchWarmupCallback (vbId=<optimized out>, fetches=..., arg=0x7fbbabffe830) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/warmup.cc:370
|
bg_itm_ctx = <optimized out>
|
items2fetch = {
|
_M_h = {<std::__detail::_Hashtable_base<StoredDocKey, std::pair<StoredDocKey const, vb_bgfetch_item_ctx_t>, std::__detail::_Select1st, std::equal_to<StoredDocKey>, std::hash<StoredDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Hashtable_traits<true, false, true> >> = {<std::__detail::_Hash_code_base<StoredDocKey, std::pair<StoredDocKey const, vb_bgfetch_item_ctx_t>, std::__detail::_Select1st, std::hash<StoredDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, true>> = {<std::__detail::_Hashtable_ebo_helper<0, std::__detail::_Select1st, true>> = {<std::__detail::_Select1st> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<1, std::hash<StoredDocKey>, true>> = {<std::hash<StoredDocKey>> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<2, std::__detail::_Mod_range_hashing, true>> = {<std::__detail::_Mod_range_hashing> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<0, std::equal_to<StoredDocKey>, true>> = {<std::equal_to<StoredDocKey>> = {<std::binary_function<StoredDocKey, StoredDocKey, bool>> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Map_base<StoredDocKey, std::pair<StoredDocKey const, vb_bgfetch_item_ctx_t>, std::allocator<std::pair<StoredDocKey const, vb_bgfetch_item_ctx_t> >, std::__detail::_Select1st, std::equal_to<StoredDocKey>, std::hash<StoredDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, true>> = {<No data fields>}, <std::__detail::_Insert<StoredDocKey, std::pair<StoredDocKey const, vb_bgfetch_item_ctx_t>, std::allocator<std::pair<StoredDocKey const, vb_bgfetch_item_ctx_t> >, std::__detail::_Select1st, std::equal_to<StoredDocKey>, std::hash<StoredDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, false>> = {<std::__detail::_Insert_base<StoredDocKey, std::pair<StoredDocKey const, vb_bgfetch_item_ctx_t>, std::allocator<std::pair<StoredDocKey const, vb_bgfetch_item_ctx_t> >, std::__detail::_Select1st, std::equal_to<StoredDocKey>, std::hash<StoredDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true> >> = {<No data fields>}, <No data fields>}, <std::__detail::_Rehash_base<StoredDocKey, std::pair<StoredDocKey const, vb_bgfetch_item_ctx_t>, std::allocator<std::pair<StoredDocKey const, vb_bgfetch_item_ctx_t> >, std::__detail::_Select1st, std::equal_to<StoredDocKey>, std::hash<StoredDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, std::integral_constant<bool, true> >> = {<No data fields>}, <std::__detail::_Equality<StoredDocKey, std::pair<StoredDocKey const, vb_bgfetch_item_ctx_t>, std::allocator<std::pair<StoredDocKey const, vb_bgfetch_item_ctx_t> >, std::__detail::_Select1st, std::equal_to<StoredDocKey>, std::hash<StoredDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, true>> = {<No data fields>}, <std::__detail::_Hashtable_alloc<std::allocator<std::__detail::_Hash_node<std::pair<StoredDocKey const, vb_bgfetch_item_ctx_t>, true> > >> = {<std::__detail::_Hashtable_ebo_helper<0, std::allocator<std::__detail::_Hash_node<std::pair<StoredDocKey const, vb_bgfetch_item_ctx_t>, true> >, true>> = {<std::allocator<std::__detail::_Hash_node<std::pair<StoredDocKey const, vb_bgfetch_item_ctx_t>, true> >> = {<__gnu_cxx::new_allocator<std::__detail::_Hash_node<std::pair<StoredDocKey const, vb_bgfetch_item_ctx_t>, true> >> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, _M_buckets = 0x7fbc279f79e0, _M_bucket_count = 3, _M_before_begin = {_M_nxt = 0x7fbba850d9b0}, _M_element_count = 1,
|
_M_rehash_policy = {static _S_growth_factor = 2, _M_max_load_factor = 1, _M_next_resize = 3}, _M_single_bucket = 0x0}}
|
applyItem = <optimized out>
|
c = 0x7fbbabffe830
|
#4 0x00007fbc574d41ce in MutationLogHarvester::apply (this=this@entry=0x7fbbabffe900, arg=arg@entry=0x7fbbabffe830,
|
mlc=mlc@entry=0x7fbc57504f30 <batchWarmupCallback(uint16_t, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > const&, void*)>)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/mutation_log.cc:1070
|
No locals.
|
#5 0x00007fbc57509ac4 in Warmup::doWarmup (this=this@entry=0x7fbc25e6c800, lf=..., vbmap=..., cb=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/warmup.cc:1147
|
start = <optimized out>
|
apply_start = <optimized out>
|
harvester = {mlog = @0x7fbc25e6c398, engine = 0x7fbc5344f800, vbid_set = {_M_t = {
|
_M_impl = {<std::allocator<std::_Rb_tree_node<unsigned short> >> = {<__gnu_cxx::new_allocator<std::_Rb_tree_node<unsigned short> >> = {<No data fields>}, <No data fields>}, <std::_Rb_tree_key_compare<std::less<unsigned short> >> = {_M_key_compare = {<std::binary_function<unsigned short, unsigned short, bool>> = {<No data fields>}, <No data fields>}}, <std::_Rb_tree_header> = {_M_header = {
|
_M_color = std::_S_red, _M_parent = 0x7fbc240b6c60, _M_left = 0x7fbc2431dd30, _M_right = 0x7fbc240b7dd0}, _M_node_count = 125}, <No data fields>}}}, committed = {
|
_M_h = {<std::__detail::_Hashtable_base<unsigned short, std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > >, std::__detail::_Select1st, std::equal_to<unsigned short>, std::hash<unsigned short>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Hashtable_traits<false, false, true> >> = {<std::__detail::_Hash_code_base<unsigned short, std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > >, std::__detail::_Select1st, std::hash<unsigned short>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, false>> = {<std::__detail::_Hashtable_ebo_helper<0, std::__detail::_Select1st, true>> = {<std::__detail::_Select1st> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<1, std::hash<unsigned short>, true>> = {<std::hash<unsigned short>> = {<std::__hash_base<unsigned long, unsigned short>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<2, std::__detail::_Mod_range_hashing, true>> = {<std::__detail::_Mod_range_hashing> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<0, std::equal_to<unsigned short>, true>> = {<std::equal_to<unsigned short>> = {<std::binary_function<unsigned short, unsigned short, bool>> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Map_base<unsigned short, std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > >, std::allocator<std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > > >, std::__detail::_Select1st, std::equal_to<unsigned short>, std::hash<unsigned short>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true>, true>> = {<No data fields>}, <std::__detail::_Insert<unsigned short, std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > >, std::allocator<std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > > >, std::__detail::_Select1st, std::equal_to<unsigned short>, std::hash<unsigned short>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true>, false>> = {<std::__detail::_Insert_base<unsigned short, std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > >, std::allocator<std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > > >, std::__detail::_Select1st, std::equal_to<unsigned short>, std::hash<unsigned short>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >> = {<No data fields>}, <No data fields>}, <std::__detail::_Rehash_base<unsigned short, std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > >, std::allocator<std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > > >, std::__detail::_Select1st, std::equal_to<unsigned short>, std::hash<unsigned short>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true>, std::integral_constant<bool, true> >> = {<No data fields>}, <std::__detail::_Equality<unsigned short, std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > >, std::allocator<std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > > >, std::__detail::_Select1st, std::equal_to<unsigned short>, std::hash<unsigned short>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true>, true>> = {<No data fields>}, <std::__detail::_Hashtable_alloc<std::allocator<std::__detail::_Hash_node<std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > >, false> > >> = {<std::__detail::_Hashtable_ebo_helper<0, std::allocator<std::__detail::_Hash_node<std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > >, false> >, true>> = {<std::allocator<std::__detail::_Hash_node<std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > >, false> >> = {<__gnu_cxx::new_allocator<std::__detail::_Hash_node<std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > >, false> >> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, _M_buckets = 0x7fbc2437e400, _M_bucket_count = 167, _M_before_begin = {
|
_M_nxt = 0x7fbba80992c0}, _M_element_count = 62, _M_rehash_policy = {static _S_growth_factor = 2, _M_max_load_factor = 1, _M_next_resize = 167}, _M_single_bucket = 0x0}}, loading = {
|
_M_h = {<std::__detail::_Hashtable_base<unsigned short, std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > >, std::__detail::_Select1st, std::equal_to<unsigned short>, std::hash<unsigned short>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Hashtable_traits<false, false, true> >> = {<std::__detail::_Hash_code_base<unsigned short, std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > >, std::__detail::_Select1st, std::hash<unsigned short>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, false>> = {<std::__detail::_Hashtable_ebo_helper<0, std::__detail::_Select1st, true>> = {<std::__detail::_Select1st> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<1, std::hash<unsigned short>, true>> = {<std::hash<unsigned short>> = {<std::__hash_base<unsigned long, unsigned short>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<2, std::__detail::_Mod_range_hashing, true>> = {<std::__detail::_Mod_range_hashing> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<0, std::equal_to<unsigned short>, true>> = {<std::equal_to<unsigned short>> = {<std::binary_function<unsigned short, unsigned short, bool>> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Map_base<unsigned short, std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > >, std::allocator<std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > > >, std::__detail::_Select1st, std::equal_to<unsigned short>, std::hash<unsigned short>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true>, true>> = {<No data fields>}, <std::__detail::_Insert<unsigned short, std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > >, std::allocator<std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > > >, std::__detail::_Select1st, std::equal_to<unsigned short>, std::hash<unsigned short>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true>, false>> = {<std::__detail::_Insert_base<unsigned short, std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > >, std::allocator<std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > > >, std::__detail::_Select1st, std::equal_to<unsigned short>, std::hash<unsigned short>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >> = {<No data fields>}, <No data fields>}, <std::__detail::_Rehash_base<unsigned short, std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > >, std::allocator<std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > > >, std::__detail::_Select1st, std::equal_to<unsigned short>, std::hash<unsigned short>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true>, std::integral_constant<bool, true> >> = {<No data fields>}, <std::__detail::_Equality<unsigned short, std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > >, std::allocator<std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > > >, std::__detail::_Select1st, std::equal_to<unsigned short>, std::hash<unsigned short>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true>, true>> = {<No data fields>}, <std::__detail::_Hashtable_alloc<std::allocator<std::__detail::_Hash_node<std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > >, false> > >> = {<std::__detail::_Hashtable_ebo_helper<0, std::allocator<std::__detail::_Hash_node<std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > >, false> >, true>> = {<std::allocator<std::__detail::_Hash_node<std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > >, false> >> = {<__gnu_cxx::new_allocator<std::__detail::_Hash_node<std::pair<unsigned short const, std::set<StoredDocKey, std::less<StoredDocKey>, std::allocator<StoredDocKey> > >, false> >> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, _M_buckets = 0x7fbbabffe9a8, _M_bucket_count = 1, _M_before_begin = {
|
_M_nxt = 0x0}, _M_element_count = 0, _M_rehash_policy = {static _S_growth_factor = 2, _M_max_load_factor = 1, _M_next_resize = 0}, _M_single_bucket = 0x0}}, itemsSeen = {1140000, 0, 0, 1137,
|
1137}}
|
it = <optimized out>
|
log_load_duration = {__r = 1424057299}
|
log_apply_duration = {__r = 443000430}
|
cookie = {cb = @0x7fbbabffea50, epstore = 0x7fbc273c8980, loaded = 0, skipped = 0, error = 0}
|
alog_iter = {<std::iterator<std::input_iterator_tag, MutationLogEntryV2 const, long, MutationLogEntryV2 const*, MutationLogEntryV2 const&>> = {<No data fields>}, log = 0x7fbc25e6c398,
|
entryBuf = {<std::_Vector_base<unsigned char, std::allocator<unsigned char> >> = {
|
_M_impl = {<std::allocator<unsigned char>> = {<__gnu_cxx::new_allocator<unsigned char>> = {<No data fields>}, <No data fields>}, _M_start = 0x7fbc2404f200 <Address 0x7fbc2404f200 out of bounds>,
|
_M_finish = 0x7fbc2404f400 <Address 0x7fbc2404f400 out of bounds>, _M_end_of_storage = 0x7fbc2404f400 <Address 0x7fbc2404f400 out of bounds>}}, <No data fields>},
|
buf = {<std::_Vector_base<unsigned char, std::allocator<unsigned char> >> = {
|
_M_impl = {<std::allocator<unsigned char>> = {<__gnu_cxx::new_allocator<unsigned char>> = {<No data fields>}, <No data fields>}, _M_start = 0x7fbc240c9000 <Address 0x7fbc240c9000 out of bounds>,
|
_M_finish = 0x7fbc240ca000 <Address 0x7fbc240ca000 out of bounds>, _M_end_of_storage = 0x7fbc240ca000 <Address 0x7fbc240ca000 out of bounds>}}, <No data fields>}, p = {
|
_M_current = 0x7fbc240c9130 <Address 0x7fbc240c9130 out of bounds>}, offset = 57655296, items = 75, isEnd = false}
|
total = <optimized out>
|
#6 0x00007fbc5750a0c0 in Warmup::loadingAccessLog (this=0x7fbc25e6c800, shardId=1) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/warmup.cc:1068
|
load_cb = {<StatusCallback<GetValue>> = {<Callback<GetValue>> = {_vptr.Callback = 0x7fbc577cc150 <vtable for LoadStorageKVPairCallback+16>}, myStatus = 0}, vbuckets = @0x7fbc273c89a0,
|
stats = @0x7fbc5344fb00, epstore = @0x7fbc273c8980, startTime = 1525140561, hasPurged = false, maybeEnableTraffic = true, warmupState = WarmupState::LoadingAccessLog}
|
success = false
|
#7 0x00007fbc5750d4c7 in WarmupLoadAccessLog::run (this=0x7fbc24078f10) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/warmup.cc:243
|
phosphor_internal_nme_242 = 0x7fbc57575216 "WarmupLoadAccessLog"
|
phosphor_internal_category_enabled_242 = {_M_b = {_M_p = 0x0}}
|
phosphor_internal_category_enabled_temp_242 = <optimized out>
|
phosphor_internal_tpi_242 = {category = 0x19bd24 <Address 0x19bd24 out of bounds>, name = 0x1aa216 <Address 0x1aa216 out of bounds>, argument_names = {_M_elems = {
|
0x19bd41 <Address 0x19bd41 out of bounds>, 0x19bd46 <Address 0x19bd46 out of bounds>}}}
|
phosphor_internal_scoped_trace_inst_242 = {<No data fields>}
|
#8 0x00007fbc5748d449 in ExecutorThread::run (this=0x7fbc25ffd000) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/executorthread.cc:147
|
startReltime = 0
|
curTaskDescr = {<cb::sized_buffer<char const>> = {static npos = <optimized out>, buf = 0x7fbc24214600 <Address 0x7fbc24214600 out of bounds>, len = <optimized out>}, <No data fields>}
|
woketime = <optimized out>
|
scheduleOverhead = <optimized out>
|
again = <optimized out>
|
runtime = <optimized out>
|
q = 0x7fbc273dd900
|
tick = 2 '\002'
|
#9 0x00007fbc6d98d9a7 in run (this=0x7fbc25f90ba0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:59
|
No locals.
|
#10 platform_thread_wrap (arg=0x7fbc25f90ba0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:72
|
context = {_M_t = {
|
_M_t = {<std::_Tuple_impl<0, CouchbaseThread*, std::default_delete<CouchbaseThread> >> = {<std::_Tuple_impl<1, std::default_delete<CouchbaseThread> >> = {<std::_Head_base<1, std::default_delete<CouchbaseThread>, true>> = {<std::default_delete<CouchbaseThread>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, CouchbaseThread*, false>> = {
|
_M_head_impl = 0x7fbc25f90ba0}, <No data fields>}, <No data fields>}}}
|
#11 0x00007fbc6bae8dc5 in start_thread (arg=0x0) at pthread_create.c:267
|
pd = 0x0
|
now = <optimized out>
|
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {140444021290752, 7144750590099734302, 0, 140444021291456, 140444021290752, 140446068101120, -7183215904381452514, -7178852620290605282},
|
mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
|
not_first_call = <optimized out>
|
pagesize_m1 = <optimized out>
|
sp = <optimized out>
|
freesize = <optimized out>
|
#12 0x00007fbc6b81776d in sockatmark (fd=<optimized out>) at ../sysdeps/unix/sockatmark.c:29
|
answ = 0
|
#13 0x0000000000000000 in ?? ()
|
|