Details
-
Bug
-
Resolution: Cannot Reproduce
-
Critical
-
Cheshire-Cat
-
None
-
couchbase-server-7.0.0-2524.x86_64
-
Untriaged
-
1
-
Unknown
-
KV Sprint 2020-Oct
Description
Two crashes due to gsl assert being hit Expects(hcs > proposedVBState.persistedCompletedSeqno);
(gdb) bt full
|
#0 0x00007f007cabf387 in raise () from /lib64/libc.so.6
|
No symbol table info available.
|
#1 0x00007f007cac0a78 in abort () from /lib64/libc.so.6
|
No symbol table info available.
|
#2 0x00007f007d61d15d in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/gcc-7.3.0/libstdc++-v3/libsupc++/vterminate.cc:50
|
terminating = false
|
t = <optimized out>
|
#3 0x000000000054d562 in backtrace_terminate_handler () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:86
|
No locals.
|
#4 0x00007f007d61af86 in __cxxabiv1::__terminate (handler=<optimized out>) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/libsupc++/eh_terminate.cc:47
|
No locals.
|
#5 0x00007f007d61afd1 in std::terminate () at /tmp/deploy/gcc-7.3.0/libstdc++-v3/libsupc++/eh_terminate.cc:57
|
No locals.
|
#6 0x00007f007d61b213 in __cxxabiv1::__cxa_throw (obj=obj@entry=0x7efff800e4a0, tinfo=tinfo@entry=0x7f00812477f0 <typeinfo for gsl::fail_fast>, dest=dest@entry=0x7f0080d464d0 <gsl::fail_fast::~fail_fast()>) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/libsupc++/eh_throw.cc:93
|
globals = <optimized out>
|
header = 0x7efff800e420
|
#7 0x00007f0080d53fe8 in gsl::fail_fast_assert (cond=<optimized out>, message=0x7f0080f94e80 "GSL: Precondition failure at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc: 665") at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/gsl-lite/include/gsl/gsl-lite.h:473
|
No locals.
|
#8 0x00007f0080dde8f3 in EPBucket::flushVBucket (this=0x7f0071491000, vbid=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:665
|
vb = {vb = {<std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<VBucket, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f0036cd7c00, _M_refcount = {_M_pi = 0x7f0036d6b700}}, <No data fields>}, lock = {_M_device = 0x7f003648d288, _M_owns = true}}
|
toFlush = {items = {<std::_Vector_base<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, std::allocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> > > >> = {
|
_M_impl = {<std::allocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> > >> = {<__gnu_cxx::new_allocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> > >> = {<No data fields>}, <No data fields>}, _M_start = 0x7f0048672c58, _M_finish = 0x7f0048672c60, _M_end_of_storage = 0x7f0048672c60}}, <No data fields>},
|
ranges = {<std::_Vector_base<CheckpointSnapshotRange, std::allocator<CheckpointSnapshotRange> >> = {_M_impl = {<std::allocator<CheckpointSnapshotRange>> = {<__gnu_cxx::new_allocator<CheckpointSnapshotRange>> = {<No data fields>}, <No data fields>}, _M_start = 0x7f003aaa1d70, _M_finish = 0x7f003aaa1da0,
|
_M_end_of_storage = 0x7f003aaa1da0}}, <No data fields>}, moreAvailable = false, maxDeletedRevSeqno = {<std::_Optional_base<unsigned long>> = {_M_payload = {{_M_empty = {<No data fields>}, _M_payload = 2},
|
_M_engaged = true}}, <std::_Enable_copy_move<true, true, true, true, std::optional<unsigned long> >> = {<No data fields>}, <No data fields>}, checkpointType = Memory, flushHandle = {_M_t = {
|
_M_t = {<std::_Tuple_impl<0, FlushHandle*, std::default_delete<FlushHandle> >> = {<std::_Tuple_impl<1, std::default_delete<FlushHandle> >> = {<std::_Head_base<1, std::default_delete<FlushHandle>, true>> = {<std::default_delete<FlushHandle>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, FlushHandle*, false>> = {
|
_M_head_impl = 0x7f00388b80e0}, <No data fields>}, <No data fields>}}}}
|
moreAvailable = EPBucket::No
|
wakeupCheckpointRemover = EPBucket::No
|
range = {<std::_Optional_base<snapshot_range_t>> = {_M_payload = {{_M_empty = {<No data fields>}, _M_payload = {start = 3105, end = 3105}}, _M_engaged = true}}, <std::_Enable_copy_move<true, true, true, true, std::optional<snapshot_range_t> >> = {<No data fields>}, <No data fields>}
|
prev = 0x7f003a8d3f60
|
persistedVbState = <optimized out>
|
vbstate = {static CurrentVersion = 3, maxDeletedSeqno = {counter = {_M_elems = "\002\000\000\000\000"}}, highSeqno = 3104, purgeSeqno = 0, lastSnapStart = 3104, lastSnapEnd = 3104, maxCas = 1594089648088481792, hlcCasEpochSeqno = 1, mightContainXattrs = false, supportsNamespaces = true, version = 3, persistedCompletedSeqno = 3519,
|
persistedPreparedSeqno = 3104, highPreparedSeqno = 3104, maxVisibleSeqno = 2611, onDiskPrepares = 990, checkpointType = Memory, transition = {failovers = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
|
_M_p = 0x7f003c41dd80 <Address 0x7f003c41dd80 out of bounds>}, _M_string_length = 370, {_M_local_buf = "r\001\000\000\000\000\000\000\067\032\203\177\000\177\000", _M_allocated_capacity = 370}}, replicationTopology = {m_type = nlohmann::detail::array, m_value = {object = 0x7f003a79efc0, array = 0x7f003a79efc0, string = 0x7f003a79efc0,
|
boolean = 192, number_integer = 139638957797312, number_unsigned = 139638957797312, number_float = 6.8990811868727523e-310}}, state = vbucket_state_active}}
|
commitData = {collections = {needsPurge = false, mutated = {
|
_M_h = {<std::__detail::_Hashtable_base<CollectionID, CollectionID, std::__detail::_Identity, std::equal_to<CollectionID>, std::hash<CollectionID>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Hashtable_traits<true, true, true> >> = {<std::__detail::_Hash_code_base<CollectionID, CollectionID, std::__detail::_Identity, std::hash<CollectionID>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, true>> = {<std::__detail::_Hashtable_ebo_helper<0, std::__detail::_Identity, true>> = {<std::__detail::_Identity> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<1, std::hash<CollectionID>, true>> = {<std::hash<CollectionID>> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<2, std::__detail::_Mod_range_hashing, true>> = {<std::__detail::_Mod_range_hashing> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<0, std::equal_to<CollectionID>, true>> = {<std::equal_to<CollectionID>> = {<std::binary_function<CollectionID, CollectionID, bool>> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Map_base<CollectionID, CollectionID, std::allocator<CollectionID>, std::__detail::_Identity, std::equal_to<CollectionID>, std::hash<CollectionID>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, true, true>, true>> = {<No data fields>}, <std::__detail::_Insert<CollectionID, CollectionID, std::allocator<CollectionID>, std::__detail::_Identity, std::equal_to<CollectionID>, std::hash<CollectionID>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, true, true>, true>> = {<std::__detail::_Insert_base<CollectionID, CollectionID, std::allocator<CollectionID>, std::__detail::_Identity, std::equal_to<CollectionID>, std::hash<CollectionID>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, true, true> >> = {<No data fields>}, <No data fields>}, <std::__detail::_Rehash_base<CollectionID, CollectionID, std::allocator<CollectionID>, std::__detail::_Identity, std::equal_to<CollectionID>, std::hash<CollectionID>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, true, true>, std::integral_constant<bool, true> >> = {<No data fields>}, <std::__detail::_Equality<CollectionID, CollectionID, std::allocator<CollectionID>, std::__detail::_Identity, std::equal_to<CollectionID>, std::hash<CollectionID>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, true, true>, true>> = {<No data fields>}, <std::__detail::_Hashtable_alloc<std::allocator<std::__detail::_Hash_node<CollectionID, true> > >> = {<std::__detail::_Hashtable_ebo_helper<0, std::allocator<std::__detail::_Hash_node<CollectionID, true> >, true>> = {<std::allocator<std::__detail::_Hash_node<CollectionID, true> >> = {<__gnu_cxx::new_allocator<std::__detail::_Hash_node<CollectionID, true> >> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, _M_buckets = 0x7f0032a44e48, _M_bucket_count = 1, _M_before_begin = {_M_nxt = 0x0}, _M_element_count = 0, _M_rehash_policy = {static _S_growth_factor = 2,
|
_M_max_load_factor = 1, _M_next_resize = 0}, _M_single_bucket = 0x0}}, manifest = @0x7f0036b1f8a0}, proposedVBState = {static CurrentVersion = 3, maxDeletedSeqno = {counter = {_M_elems = "\002\000\000\000\000"}}, highSeqno = 3104, purgeSeqno = 0, lastSnapStart = 3105, lastSnapEnd = 3105, maxCas = 1594089702109872128, hlcCasEpochSeqno = 1,
|
mightContainXattrs = false, supportsNamespaces = true, version = 3, persistedCompletedSeqno = 3519, persistedPreparedSeqno = 3104, highPreparedSeqno = 3104, maxVisibleSeqno = 2611, onDiskPrepares = 990, checkpointType = Memory, transition = {failovers = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f003c41e500 <Address 0x7f003c41e500 out of bounds>}, _M_string_length = 370, {_M_local_buf = "r\001\000\000\000\000\000\000\200;\002s\000\177\000", _M_allocated_capacity = 370}}, replicationTopology = {
|
m_type = nlohmann::detail::array, m_value = {object = 0x7f00702feee0, array = 0x7f00702feee0, string = 0x7f00702feee0, boolean = 224, number_integer = 139639858917088, number_unsigned = 139639858917088, number_float = 6.8991257081051633e-310}}, state = vbucket_state_active}}}
|
proposedVBState = @0x7f0032a44e58: {static CurrentVersion = 3, maxDeletedSeqno = {counter = {_M_elems = "\002\000\000\000\000"}}, highSeqno = 3104, purgeSeqno = 0, lastSnapStart = 3105, lastSnapEnd = 3105, maxCas = 1594089702109872128, hlcCasEpochSeqno = 1, mightContainXattrs = false, supportsNamespaces = true, version = 3,
|
persistedCompletedSeqno = 3519, persistedPreparedSeqno = 3104, highPreparedSeqno = 3104, maxVisibleSeqno = 2611, onDiskPrepares = 990, checkpointType = Memory, transition = {failovers = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
|
_M_p = 0x7f003c41e500 <Address 0x7f003c41e500 out of bounds>}, _M_string_length = 370, {_M_local_buf = "r\001\000\000\000\000\000\000\200;\002s\000\177\000", _M_allocated_capacity = 370}}, replicationTopology = {m_type = nlohmann::detail::array, m_value = {object = 0x7f00702feee0, array = 0x7f00702feee0, string = 0x7f00702feee0,
|
boolean = 224, number_integer = 139639858917088, number_unsigned = 139639858917088, number_float = 6.8991257081051633e-310}}, state = vbucket_state_active}}
|
maxSeqno = 3105
|
maxVbStateOpCas = 0
|
minSeqno = 3105
|
flushBatchSize = 1
|
mustPersistVBState = false
|
hcs = {<std::_Optional_base<unsigned long>> = {_M_payload = {{_M_empty = {<No data fields>}, _M_payload = 3103}, _M_engaged = true}}, <std::_Enable_copy_move<true, true, true, true, std::optional<unsigned long> >> = {<No data fields>}, <No data fields>}
|
hps = {<std::_Optional_base<unsigned long>> = {_M_payload = {{_M_empty = {<No data fields>}, _M_payload = 0}, _M_engaged = false}}, <std::_Enable_copy_move<true, true, true, true, std::optional<unsigned long> >> = {<No data fields>}, <No data fields>}
|
maxVisibleSeqno = {<ThrowExceptionPolicy<unsigned long>> = {label = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f0032a44ce0 "unlabelled"}, _M_string_length = 10, {_M_local_buf = "unlabelled\000\000\000\000\000",
|
_M_allocated_capacity = 7812730926796664437}}}, val = 2611}
|
aggStats = {numItems = 1, totalBytes = 134, totalAgeInMicro = 0}
|
flushSuccess = <optimized out>
|
highSeqno = <optimized out>
|
#9 0x00007f0080e33c0c in Flusher::flushVB (this=this@entry=0x7f0035fcf340) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:306
|
vbid = {vbid = 733}
|
res = {moreAvailable = EPBucket::No, numFlushed = 0, wakeupCkptRemover = EPBucket::No}
|
#10 0x00007f0080e34dd9 in Flusher::step (this=0x7f0035fcf340, task=0x7f0037dc6e70) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:207
|
more = <optimized out>
|
#11 0x00007f0080e37d83 in GlobalTask::execute (this=0x7f0037dc6e70) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/globaltask.cc:73
|
guard = {previous = 0x0}
|
#12 0x00007f0080e2e5af in ExecutorThread::run (this=0x7f007b5ec5e0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/executorthread.cc:188
|
curTaskDescr = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f007b78f000 <Address 0x7f007b78f000 out of bounds>}, _M_string_length = 31, {_M_local_buf = "\037\000\000\000\000\000\000\000^#\000\000\000\000\000",
|
_M_allocated_capacity = 31}}
|
woketime = <optimized out>
|
scheduleOverhead = <optimized out>
|
again = <optimized out>
|
runtime = <optimized out>
|
q = <optimized out>
|
tick = 68 'D'
|
guard = {engine = 0x0}
|
#13 0x00007f007f821777 in run (this=0x7f0073350f30) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58
|
No locals.
|
#14 platform_thread_wrap (arg=0x7f0073350f30) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71
|
context = {_M_t = {
|
_M_t = {<std::_Tuple_impl<0, CouchbaseThread*, std::default_delete<CouchbaseThread> >> = {<std::_Tuple_impl<1, std::default_delete<CouchbaseThread> >> = {<std::_Head_base<1, std::default_delete<CouchbaseThread>, true>> = {<std::default_delete<CouchbaseThread>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, CouchbaseThread*, false>> = {_M_head_impl = 0x7f0073350f30}, <No data fields>}, <No data fields>}}}
|
#15 0x00007f007ce5eea5 in start_thread () from /lib64/libpthread.so.0
|
No symbol table info available.
|
#16 0x00007f007cb878dd in clone () from /lib64/libc.so.6
|
./testrunner -i /tmp/win10-bucket-ops.ini rerun=False,crash_warning=True,quota_percent=95 -t failover.AutoFailoverTests.AutoFailoverTests.test_autofailover_during_rebalance,timeout=5,num_node_failures=1,nodes_in=0,nodes_out=1,failover_action=restart_server,nodes_init=4,can_abort_rebalance=False,override_spec_params=durability,durability=MAJORITY,bucket_spec=single_bucket.def_scope_fifty_collections,GROUP=auto_failover_durability
|
Supportal link:
https://supportal.couchbase.com/snapshot/642be23f540f4ffc2fe484ccdc04c0b2::0
Cbcollect_info:
s3://cb-customers-secure/mb-40286/mb-40286/2020-07-07/collectinfo-2020-07-07t065843-ns_1@172.23.104.186.zip
s3://cb-customers-secure/mb-40286/mb-40286/2020-07-07/collectinfo-2020-07-07t065843-ns_1@172.23.120.201.zip
s3://cb-customers-secure/mb-40286/mb-40286/2020-07-07/collectinfo-2020-07-07t065843-ns_1@172.23.120.206.zip
s3://cb-customers-secure/mb-40286/mb-40286/2020-07-07/collectinfo-2020-07-07t065843-ns_1@172.23.121.10.zip
s3://cb-customers-secure/mb-40286/mb-40286/2020-07-07/collectinfo-2020-07-07t065843-ns_1@172.23.98.195.zip
s3://cb-customers-secure/mb-40286/mb-40286/2020-07-07/collectinfo-2020-07-07t065843-ns_1@172.23.98.196.zip
Pcaps
pcaps.zip
Automation Logs
Automation_logs.zip
Few Cores of interest that has GSL: Precondition failure(There would be many more)
172.23.98.196 : Stack Trace of 8fe856cb-ee53-4118-cf28c2b1-c16bfd0e.dmp
172.23.98.196 : Stack Trace of 559111e6-41dd-4a99-bf0e7b8d-870f3f4e.dmp
Attachments
Issue Links
- relates to
-
MB-40286 VBucket::commit (vb:924) failed as no HashTable item found with key
- Closed