Details
-
Bug
-
Resolution: Duplicate
-
Major
-
Cheshire-Cat
-
Ubuntu 1804
Couchbase version 7.0.0-2812
-
Untriaged
-
1
-
Unknown
Description
Crash in memcached during kv-engine-cheshire-cat-jepsen-nightly-229, test lein trampoline run test --nodes-file ./nodes --username root --password couchbase --workload=disk-failure --manipulate-disks --replicas=1 --durability=0:0:100:0 --recovery-type=delta --eviction-policy=full --cas --enable-tcp-capture --hashdump --enable-memcached-debug-log-level --net-interface=eth0
(gdb) bt
|
#0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
|
#1 0x00007fdbd14cd8b1 in __GI_abort () at abort.c:79
|
#2 0x00007fdbd20eb165 in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/vterminate.cc:95
|
#3 0x000000000054ec22 in backtrace_terminate_handler () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:86
|
#4 0x00007fdbd20e8f56 in __cxxabiv1::__terminate (handler=<optimized out>) at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/eh_terminate.cc:47
|
#5 0x00007fdbd20e8fa1 in std::terminate () at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/eh_terminate.cc:57
|
#6 0x00007fdbd20e91e3 in __cxxabiv1::__cxa_throw (obj=obj@entry=0x7fdb88000e50, tinfo=0x7fdbd23ccad8 <typeinfo for std::underflow_error>, dest=0x7fdbd20fe460 <std::underflow_error::~underflow_error()>) at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/eh_throw.cc:93
|
#7 0x00007fdbd585b534 in cb::ThrowExceptionUnderflowPolicy<unsigned long>::underflow (this=<optimized out>, desired=<optimized out>, current=<optimized out>, arg=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/include/platform/non_negative_counter.h:48
|
#8 0x00007fdbd5a27ef1 in cb::NonNegativeCounter<unsigned long, cb::ThrowExceptionUnderflowPolicy>::fetch_add (arg=-1, this=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/include/platform/non_negative_counter.h:125
|
#9 cb::NonNegativeCounter<unsigned long, cb::ThrowExceptionUnderflowPolicy>::operator+= (rhs=18446744073709551615, this=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/include/platform/non_negative_counter.h:182
|
#10 Collections::VB::ManifestEntry::updateDiskSize (delta=-1, this=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/collections/vbucket_manifest_entry.h:109
|
#11 Collections::VB::Manifest::updateDiskSize (delta=-1, entry=..., this=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/collections/vbucket_manifest.h:1095
|
#12 Collections::VB::Manifest::CachingReadHandle::updateDiskSize (delta=-1, this=0x7fdb967f1210) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/collections/vbucket_manifest.h:407
|
#13 Collections::VB::Flush::updateDiskSize (this=<optimized out>, key=..., delta=-1) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/collections/flush.cc:59
|
#14 0x00007fdbd59edd82 in saveDocsCallback (oldInfo=0x7fdb8f505470, newInfo=0x7fdb8f505400, context=0x7fdb967f1990) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/couch-kvstore/couch-kvstore.cc:2191
|
#15 0x00007fdbd55903ab in do_save_callback (valueNew=<optimized out>, valueOld=0x7fdb967f13c0, key=0x7fdb942ec740, rq=0x7fdb967f1590) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/btree_modify.cc:431
|
#16 modify_node (rq=rq@entry=0x7fdb967f1590, nptr=nptr@entry=0x7fdb943d83c0, start=start@entry=0, end=2, dst=dst@entry=0x7fdbc4514010) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/btree_modify.cc:537
|
#17 0x00007fdbd5590d19 in modify_btree (rq=rq@entry=0x7fdb967f1590, root=0x7fdb943d83c0, errcode=errcode@entry=0x7fdb967f14c4) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/btree_modify.cc:809
|
#18 0x00007fdbd55a4de0 in update_indexes (save_callback_ctx=<optimized out>, save_callback=<optimized out>, numdocs=<optimized out>, idvals=<optimized out>, ids=<optimized out>, seqvals=<optimized out>, seqs=0x7fdb942ec710, db=<optimized out>, this=<optimized out>, this=<optimized out>)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/couch_save.cc:238
|
#19 couchstore_save_documents_and_callback (db=<optimized out>, docs=<optimized out>, infos=<optimized out>, numdocs=<optimized out>, options=options@entry=3, save_cb=save_cb@entry=0x7fdbd59edc70 <saveDocsCallback(DocInfo const*, DocInfo const*, void*)>, save_cb_ctx=<optimized out>)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/couch_save.cc:424
|
#20 0x00007fdbd59fad58 in CouchKVStore::saveDocs (this=this@entry=0x7fdbc66a4c00, vbid=..., docs=..., docinfos=..., kvctx=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/couch-kvstore/couch-kvstore.cc:2241
|
#21 0x00007fdbd59fbc8a in CouchKVStore::commit2couchstore (this=0x7fdbc66a4c00, commitData=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/couch-kvstore/couch-kvstore.cc:2095
|
#22 0x00007fdbd59fc325 in CouchKVStore::commit (this=0x7fdbc66a4c00, commitData=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/couch-kvstore/couch-kvstore.cc:1313
|
#23 0x00007fdbd58ec7e2 in EPBucket::commit (this=this@entry=0x7fdbc6690000, vbid=..., kvstore=..., commitData=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:863
|
#24 0x00007fdbd58effbd in EPBucket::flushVBucket (this=0x7fdbc6690000, vbid=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:732
|
#25 0x00007fdbd593f71c in Flusher::flushVB (this=this@entry=0x7fdbc66dd8c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:306
|
#26 0x00007fdbd5940649 in Flusher::step (this=0x7fdbc66dd8c0, task=0x7fdbc669f330) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:207
|
#27 0x00007fdbd59435f3 in GlobalTask::execute (this=0x7fdbc669f330) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/globaltask.cc:73
|
#28 0x00007fdbd58555af in CB3ExecutorThread::run (this=0x7fdbcde206c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/cb3_executorthread.cc:188
|
#29 0x00007fdbd42f6387 in CouchbaseThread::run (this=0x7fdbd0e3d040) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58
|
#30 platform_thread_wrap (arg=0x7fdbd0e3d040) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71
|
#31 0x00007fdbd18856db in start_thread (arg=0x7fdb967fc700) at pthread_create.c:463
|
#32 0x00007fdbd15aea3f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
|
(gdb) bt full
|
#0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
|
set = {__val = {0, 140582089889539, 140580856270304, 0, 140582089894064, 140581099473440, 386183668047020927, 3524339112, 2524909848, 281470681751456, 0, 0, 0, 0, 0, 0}}
|
pid = <optimized out>
|
tid = <optimized out>
|
ret = <optimized out>
|
#1 0x00007fdbd14cd8b1 in __GI_abort () at abort.c:79
|
save_stage = 1
|
act = {__sigaction_handler = {sa_handler = 0xfffdd6, sa_sigaction = 0xfffdd6}, sa_mask = {__val = {16777050, 0, 0, 0, 0, 140582089889539, 140582086352833, 11, 140582089889408, 10, 140580856270448, 140581099475280, 140580856270416, 140581099475344, 140582086354035, 140582089889408}}, sa_flags = -770801328, sa_restorer = 0x7fdbc4498c28}
|
sigs = {__val = {32, 0 <repeats 15 times>}}
|
__cnt = <optimized out>
|
__set = <optimized out>
|
__cnt = <optimized out>
|
__set = <optimized out>
|
#2 0x00007fdbd20eb165 in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/vterminate.cc:95
|
terminating = false
|
t = <optimized out>
|
#3 0x000000000054ec22 in backtrace_terminate_handler () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:86
|
No locals.
|
#4 0x00007fdbd20e8f56 in __cxxabiv1::__terminate (handler=<optimized out>) at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/eh_terminate.cc:47
|
No locals.
|
#5 0x00007fdbd20e8fa1 in std::terminate () at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/eh_terminate.cc:57
|
No locals.
|
#6 0x00007fdbd20e91e3 in __cxxabiv1::__cxa_throw (obj=obj@entry=0x7fdb88000e50, tinfo=0x7fdbd23ccad8 <typeinfo for std::underflow_error>, dest=0x7fdbd20fe460 <std::underflow_error::~underflow_error()>) at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/eh_throw.cc:93
|
globals = <optimized out>
|
header = 0x7fdb88000dd0
|
#7 0x00007fdbd585b534 in cb::ThrowExceptionUnderflowPolicy<unsigned long>::underflow (this=<optimized out>, desired=<optimized out>, current=<optimized out>, arg=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/include/platform/non_negative_counter.h:48
|
No locals.
|
#8 0x00007fdbd5a27ef1 in cb::NonNegativeCounter<unsigned long, cb::ThrowExceptionUnderflowPolicy>::fetch_add (arg=-1, this=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/include/platform/non_negative_counter.h:125
|
current = <optimized out>
|
desired = 18446744073709551615
|
current = <optimized out>
|
desired = <optimized out>
|
#9 cb::NonNegativeCounter<unsigned long, cb::ThrowExceptionUnderflowPolicy>::operator+= (rhs=18446744073709551615, this=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/include/platform/non_negative_counter.h:182
|
No locals.
|
#10 Collections::VB::ManifestEntry::updateDiskSize (delta=-1, this=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/collections/vbucket_manifest_entry.h:109
|
No locals.
|
#11 Collections::VB::Manifest::updateDiskSize (delta=-1, entry=..., this=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/collections/vbucket_manifest.h:1095
|
No locals.
|
#12 Collections::VB::Manifest::CachingReadHandle::updateDiskSize (delta=-1, this=0x7fdb967f1210) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/collections/vbucket_manifest.h:407
|
No locals.
|
#13 Collections::VB::Flush::updateDiskSize (this=<optimized out>, key=..., delta=-1) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/collections/flush.cc:59
|
No locals.
|
#14 0x00007fdbd59edd82 in saveDocsCallback (oldInfo=0x7fdb8f505470, newInfo=0x7fdb8f505400, context=0x7fdb967f1990) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/couch-kvstore/couch-kvstore.cc:2191
|
cbCtx = 0x7fdb967f1990
|
newKey = {keydata = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7fdb967f12f0 ""}, _M_string_length = 11, {_M_local_buf = "\000jepsen0007\000\000\000\000", _M_allocated_capacity = 3489838307740051968}}}
|
onDiskMutationType = <optimized out>
|
docKey = {<DocKeyInterface<DocKey>> = {<No data fields>}, buffer = {static npos = <optimized out>, buf = 0x7fdb967f12f0 "", len = 11}, encoding = DocKeyEncodesCollectionId::Yes}
|
oldSize = <optimized out>
|
newSize = <optimized out>
|
delta = <optimized out>
|
#15 0x00007fdbd55903ab in do_save_callback (valueNew=<optimized out>, valueOld=0x7fdb967f13c0, key=0x7fdb942ec740, rq=0x7fdb967f1590) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/btree_modify.cc:431
|
infoNew = 0x7fdb8f505400
|
infoOld = 0x7fdb8f505470
|
infoNew = <optimized out>
|
infoOld = <optimized out>
|
#16 modify_node (rq=rq@entry=0x7fdb967f1590, nptr=nptr@entry=0x7fdb943d83c0, start=start@entry=0, end=2, dst=dst@entry=0x7fdbc4514010) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/btree_modify.cc:537
|
cmp_val = <optimized out>
|
cmp_key = {buf = 0x7fdbc44e0d46 <error: Cannot access memory at address 0x7fdbc44e0d46>, size = 11}
|
val_buf = {buf = 0x7fdbc44e0d51 <error: Cannot access memory at address 0x7fdbc44e0d51>, size = 48}
|
advance = 1
|
nodebuf = 0x7fdbc44e0d40 <error: Cannot access memory at address 0x7fdbc44e0d40>
|
bufpos = 65
|
nodebuflen = 130
|
errcode = <optimized out>
|
local_result = 0x7fdbc45140c8
|
#17 0x00007fdbd5590d19 in modify_btree (rq=rq@entry=0x7fdb967f1590, root=0x7fdb943d83c0, errcode=errcode@entry=0x7fdb967f14c4) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/btree_modify.cc:809
|
a = 0x7fdb9424c8e0
|
ret_ptr = 0x7fdb943d83c0
|
root_result = <optimized out>
|
#18 0x00007fdbd55a4de0 in update_indexes (save_callback_ctx=<optimized out>, save_callback=<optimized out>, numdocs=<optimized out>, idvals=<optimized out>, ids=<optimized out>, seqvals=<optimized out>, seqs=0x7fdb942ec710, db=<optimized out>, this=<optimized out>, this=<optimized out>)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/couch_save.cc:238
|
size = 70
|
seqrq = {cmp = {compare = 0x0}, file = 0x51964e <cb_malloc(size_t)+14>, num_actions = 2, actions = 0x72, fetch_callback = 0x7fdb967f1820, fetch_callback_ctx = 0x7fdbd5e18769 <operator new(unsigned long)+9>, reduce = 0x200,
|
rereduce = 0x42cb2c <std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_mutate(unsigned long, unsigned long, char const*, unsigned long)+236>, user_reduce_ctx = 0x7fdb967f18a8, purge_kp = 0x68, purge_kv = 0x7fdbcdec5e00, enable_purging = 1, guided_purge_ctx = 0x7fdb967f16c0, compacting = -1770055280,
|
kv_chunk_threshold = 32731, kp_chunk_threshold = -1770055376, save_callback = 0xaaaaaaaaaaaaaaab, save_callback_ctx = 0x7fdb967f16e0, docinfo_callback = 0x7fdbd15be056 <__GI___clock_gettime+38>}
|
errcode = COUCHSTORE_SUCCESS
|
new_id_root = <optimized out>
|
actbuf = {_M_t = {_M_t = {<std::_Tuple_impl<0, fatbuf*, cb::couchstore::FatbufDeletor>> = {<std::_Tuple_impl<1, cb::couchstore::FatbufDeletor>> = {<std::_Head_base<1, cb::couchstore::FatbufDeletor, true>> = {<cb::couchstore::FatbufDeletor> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, fatbuf*, false>> = {
|
_M_head_impl = 0x7fdb943e0180}, <No data fields>}, <No data fields>}}}
|
seqacts = 0x7fdb943e0190
|
idacts = <optimized out>
|
sorted_ids = <optimized out>
|
fetcharg = {seqacts = 0x7fdb943e0190, actpos = 1, valpos = 0, deltermbuf = @0x7fdb943e0180}
|
idrq = {cmp = {compare = 0x7fdbd55b35a0 <ebin_cmp(sized_buf const*, sized_buf const*)>}, file = 0x7fdb946bd060, num_actions = 2, actions = 0x7fdb943e01c0, fetch_callback = 0x7fdbd55a4440 <idfetch_update_cb(couchfile_modify_request*, sized_buf*, sized_buf*, void*)>, fetch_callback_ctx = 0x7fdb967f1500, reduce = 0x7fdbd55b2c30
|
<by_id_reduce(char*, size_t*, nodelist const*, int, void*)>, rereduce = 0x7fdbd55b2d00 <by_id_rereduce(char*, size_t*, nodelist const*, int, void*)>, user_reduce_ctx = 0x0, purge_kp = 0x0, purge_kv = 0x0, enable_purging = 0, guided_purge_ctx = 0x0, compacting = 0, kv_chunk_threshold = 1279, kp_chunk_threshold = 1279,
|
save_callback = 0x7fdbd59edc70 <saveDocsCallback(DocInfo const*, DocInfo const*, void*)>, save_callback_ctx = 0x7fdb967f1990, docinfo_callback = 0x7fdbd5596800 <by_id_read_docinfo(_docinfo**, _sized_buf const*, _sized_buf const*)>}
|
new_seq_root = <optimized out>
|
size = <optimized out>
|
actbuf = <optimized out>
|
seqacts = <optimized out>
|
idacts = <optimized out>
|
sorted_ids = <optimized out>
|
fetcharg = <optimized out>
|
idrq = <optimized out>
|
errcode = <optimized out>
|
new_id_root = <optimized out>
|
seqrq = <optimized out>
|
new_seq_root = <optimized out>
|
ii = <optimized out>
|
---Type <return> to continue, or q <return> to quit---
|
ii = <optimized out>
|
isorted = <optimized out>
|
#19 couchstore_save_documents_and_callback (db=<optimized out>, docs=<optimized out>, infos=<optimized out>, numdocs=<optimized out>, options=options@entry=3, save_cb=save_cb@entry=0x7fdbd59edc70 <saveDocsCallback(DocInfo const*, DocInfo const*, void*)>, save_cb_ctx=<optimized out>)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/couch_save.cc:424
|
CL_item = {_M_t = {
|
_M_t = {<std::_Tuple_impl<0, CouchLatencyItem*, std::default_delete<CouchLatencyItem> >> = {<std::_Tuple_impl<1, std::default_delete<CouchLatencyItem> >> = {<std::_Head_base<1, std::default_delete<CouchLatencyItem>, true>> = {<std::default_delete<CouchLatencyItem>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, CouchLatencyItem*, false>> = {_M_head_impl = 0x0}, <No data fields>}, <No data fields>}}}
|
CL_item_ptr = {value = {_M_b = {
|
Attachments
Issue Links
- duplicates
-
MB-39946 [Collections] Underflow of diskSize - total size (bytes) of items in this collection on disk
- Closed