Details
-
Bug
-
Resolution: Fixed
-
Critical
-
master
-
Untriaged
-
Unknown
-
KV Sprint 2020-January
Description
If anyone hits this bug and needs a workaround, executing cbworkloadgen as follows seems to work, the bug is triggered when LD_LIBRARY_PATH is set from the main bin/cbworkloadgen wrapper
couchbase/lib/python/runtime/bin/python3 couchbase/lib/python/cbworkloadgen
|
cbworkloadgen is faulting during dlopen (initialisation), failing within free being called from folly access spreader (from our platform library).
This was suspected to be TLS changes (which are not safe for dlopen) but they have been reverted for platform and jemalloc and the crash remains
The following GDB backtrace shows that we were freeing an invalid pointer
(gdb) bt full
|
#0 rtree_szind_slab_read (r_slab=<synthetic pointer>, r_szind=<synthetic pointer>, dependent=true, key=93824997281024, rtree_ctx=<optimized out>, rtree=<optimized out>, tsdn=<optimized out>) at include/jemalloc/internal/rtree.h:500
|
No locals.
|
#1 ifree (slow_path=false, tcache=0x555555b94ef0, ptr=0x555555a24d00, tsd=0x555555b94d00) at src/jemalloc.c:2570
|
usize = <optimized out>
|
#2 je_dallocx (ptr=ptr@entry=0x555555a24d00, flags=<optimized out>) at src/jemalloc.c:3522
|
tcache = 0x555555b94ef0
|
#3 0x00007fffeed89a0a in cb::_JEArenaMalloc<cb::JEArenaCoreLocalTracker>::free (ptr=ptr@entry=0x555555a24d00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/je_arena_malloc.cc:276
|
No locals.
|
#4 0x00007fffeed7fcda in free (ptr=0x555555a24d00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/include/platform/cb_arena_malloc.h:163
|
No locals.
|
#5 cb_free (ptr=0x555555a24d00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_malloc.cc:75
|
No locals.
|
#6 0x00007fffeed951ce in deallocate (this=0x7fffffff8990, __p=<optimized out>) at /usr/local/include/c++/7.3.0/ext/new_allocator.h:125
|
No locals.
|
#7 deallocate (__a=..., __n=<optimized out>, __p=<optimized out>) at /usr/local/include/c++/7.3.0/bits/alloc_traits.h:462
|
No locals.
|
#8 _M_destroy (__size=<optimized out>, this=0x7fffffff8990) at /usr/local/include/c++/7.3.0/bits/basic_string.h:226
|
No locals.
|
#9 _M_dispose (this=0x7fffffff8990) at /usr/local/include/c++/7.3.0/bits/basic_string.h:221
|
No locals.
|
#10 ~basic_string (this=0x7fffffff8990, __in_chrg=<optimized out>) at /usr/local/include/c++/7.3.0/bits/basic_string.h:647
|
No locals.
|
#11 folly::CacheLocality::readFromProcCpuinfo () at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/concurrency/CacheLocality.cpp:265
|
str = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x555555a24d00 "vendor_id\t: GenuineIntel"}, _M_string_length = 24, {_M_local_buf = "\030\000\000\000\000\000\000\000r\t: 0\000\000", _M_allocated_capacity = 24}}
|
xi = {<std::basic_istream<char, std::char_traits<char> >> = {<std::basic_ios<char, std::char_traits<char> >> = {<std::ios_base> = {_vptr.ios_base = 0x7fffef877858 <vtable for std::basic_ifstream<char, std::char_traits<char> >+64>, static boolalpha = std::_S_boolalpha, static dec = std::_S_dec, static fixed = std::_S_fixed, static hex = std::_S_hex, static internal = std::_S_internal,
|
static left = std::_S_left, static oct = std::_S_oct, static right = std::_S_right, static scientific = std::_S_scientific, static showbase = std::_S_showbase, static showpoint = std::_S_showpoint, static showpos = std::_S_showpos, static skipws = std::_S_skipws, static unitbuf = std::_S_unitbuf, static uppercase = std::_S_uppercase, static adjustfield = std::_S_adjustfield,
|
static basefield = std::_S_basefield, static floatfield = std::_S_floatfield, static badbit = std::_S_badbit, static eofbit = std::_S_eofbit, static failbit = std::_S_failbit, static goodbit = std::_S_goodbit, static app = std::_S_app, static ate = std::_S_ate, static binary = std::_S_bin, static in = std::_S_in, static out = std::_S_out, static trunc = std::_S_trunc, static beg = std::_S_beg,
|
static cur = std::_S_cur, static end = std::_S_end, _M_precision = 6, _M_width = 0, _M_flags = 4098, _M_exception = std::_S_goodbit, _M_streambuf_state = std::_S_goodbit, _M_callbacks = 0x0, _M_word_zero = {_M_pword = 0x0, _M_iword = 0}, _M_local_word = {{_M_pword = 0x0, _M_iword = 0}, {_M_pword = 0x0, _M_iword = 0}, {_M_pword = 0x0, _M_iword = 0}, {_M_pword = 0x0, _M_iword = 0}, {_M_pword = 0x0,
|
_M_iword = 0}, {_M_pword = 0x0, _M_iword = 0}, {_M_pword = 0x0, _M_iword = 0}, {_M_pword = 0x0, _M_iword = 0}}, _M_word_size = 8, _M_word = 0x7fffffff8af0, _M_ios_locale = {static none = 0, static ctype = 1, static numeric = 2, static collate = 4, static time = 8, static monetary = 16, static messages = 32, static all = 63, _M_impl = 0x7fffef880ca0 <(anonymous namespace)::c_locale_impl>,
|
static _S_classic = 0x7fffef880ca0 <(anonymous namespace)::c_locale_impl>, static _S_global = 0x7fffef880ca0 <(anonymous namespace)::c_locale_impl>, static _S_categories = 0x7fffef872320 <__gnu_cxx::category_names>, static _S_once = 2, static _S_twinned_facets = 0x7fffef873ac0 <std::locale::_S_twinned_facets>}}, _M_tie = 0x0, _M_fill = 0 '\000', _M_fill_init = false, _M_streambuf = 0x7fffffff89c0,
|
_M_ctype = 0x7fffef8806c0 <(anonymous namespace)::ctype_c>, _M_num_put = 0x7fffef880650 <(anonymous namespace)::num_put_c>, _M_num_get = 0x7fffef880660 <(anonymous namespace)::num_get_c>}, _vptr.basic_istream = 0x7fffef877830 <vtable for std::basic_ifstream<char, std::char_traits<char> >+24>, _M_gcount = 25}, _M_filebuf = {<std::basic_streambuf<char, std::char_traits<char> >> = {
|
_vptr.basic_streambuf = 0x7fffef877738 <vtable for std::basic_filebuf<char, std::char_traits<char> >+16>, _M_in_beg = 0x555555b26070 "processor\t: 0\nvendor_id\t: GenuineIntel\ncpu family\t: 6\nmodel\t\t: 70\nmodel name\t: Intel(R) Core(TM) i7-4850HQ CPU @ 2.30GHz\nstepping\t: 1\ncpu MHz\t\t: 2294.641\ncache size\t: 6144 KB\nphysical id\t: 0\nsiblings\t"...,
|
_M_in_cur = 0x555555b26097 "cpu family\t: 6\nmodel\t\t: 70\nmodel name\t: Intel(R) Core(TM) i7-4850HQ CPU @ 2.30GHz\nstepping\t: 1\ncpu MHz\t\t: 2294.641\ncache size\t: 6144 KB\nphysical id\t: 0\nsiblings\t: 2\ncore id\t\t: 0\ncpu cores\t: 2\napicid\t\t"...,
|
_M_in_end = 0x555555b26636 "_ = [(\"buf\", ctypes.POINTER(ctypes.c_char)), (\"size\", ctypes.c_size_t)]\n\n def __init__(self, string):\n if string is not None:\n", ' ' <repeats 12 times>, "if isinstance(string, bytes):\n", ' ' <repeats 16 times>, "length"..., _M_out_beg = 0x0, _M_out_cur = 0x0, _M_out_end = 0x0, _M_buf_locale = {static none = 0, static ctype = 1, static numeric = 2,
|
static collate = 4, static time = 8, static monetary = 16, static messages = 32, static all = 63, _M_impl = 0x7fffef880ca0 <(anonymous namespace)::c_locale_impl>, static _S_classic = 0x7fffef880ca0 <(anonymous namespace)::c_locale_impl>, static _S_global = 0x7fffef880ca0 <(anonymous namespace)::c_locale_impl>, static _S_categories = 0x7fffef872320 <__gnu_cxx::category_names>, static _S_once = 2,
|
static _S_twinned_facets = 0x7fffef873ac0 <std::locale::_S_twinned_facets>}}, _M_lock = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 39 times>, __align = 0}, _M_file = {_M_cfile = 0x555555ade1a0, _M_cfile_created = true}, _M_mode = std::_S_in, _M_state_beg = {__count = 0, __value = {__wch = 0,
|
__wchb = "\000\000\000"}}, _M_state_cur = {__count = 0, __value = {__wch = 0, __wchb = "\000\000\000"}}, _M_state_last = {__count = 0, __value = {__wch = 0, __wchb = "\000\000\000"}},
|
_M_buf = 0x555555b26070 "processor\t: 0\nvendor_id\t: GenuineIntel\ncpu family\t: 6\nmodel\t\t: 70\nmodel name\t: Intel(R) Core(TM) i7-4850HQ CPU @ 2.30GHz\nstepping\t: 1\ncpu MHz\t\t: 2294.641\ncache size\t: 6144 KB\nphysical id\t: 0\nsiblings\t"..., _M_buf_size = 8192, _M_buf_allocated = true, _M_reading = true, _M_writing = false, _M_pback = 0 '\000', _M_pback_cur_save = 0x0, _M_pback_end_save = 0x0,
|
_M_pback_init = false, _M_codecvt = 0x7fffef880630 <(anonymous namespace)::codecvt_c>, _M_ext_buf = 0x0, _M_ext_buf_size = 0, _M_ext_next = 0x0, _M_ext_end = 0x0}}
|
buf = "vendor_id\t: GenuineIntel\000\304\336\367\377\177\000\000\000\000\000\000\000\000\000\000\212T\201\367\377\177\000\000\000 \377\367\377\177\000\000\202D\336\367\377\177\000\000P\000\000\000\000\000\000\000\324\304\336\367\377\177\000\000\222\216\230\064\000\000\000\000\001\000\000\000\000\000\000\000З\377\367\377\177\000\000\202D\336\367\377\177\000\000^\322,8\000\000\000\000\324\304\336\367\377\177\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000@o\267UUU\000\000\202D\336\367\377\177", '\000' <repeats 13 times>, "\200\000\000\000\000\000\320\377\367\377\177\000\000\002\000\000\000\000\000\000\000@o\267UUU\000\000\201"...
|
lines = {<std::_Vector_base<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >> = {
|
_M_impl = {<std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >> = {<__gnu_cxx::new_allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >> = {<No data fields>}, <No data fields>}, _M_start = 0x7fffed61d000, _M_finish = 0x7fffed61d020, _M_end_of_storage = 0x7fffed61d020}}, <No data fields>}
|
#12 0x00007fffeed9553d in folly::CacheLocality::system<std::atomic> () at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/concurrency/CacheLocality.cpp:39
|
numCpus = <optimized out>
|
cache = 0x0
|
#13 0x00007fffeed97895 in folly::AccessSpreader<std::atomic>::initialize () at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/concurrency/CacheLocality.h:359
|
cacheLocality = <optimized out>
|
n = 1
|
#14 0x00007fffeed7d53a in __static_initialization_and_destruction_0 (__priority=65535, __initialize_p=1) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/concurrency/CacheLocality.h:395
|
No locals.
|
#15 _GLOBAL__sub_I_CacheLocality.cpp(void) () at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/concurrency/CacheLocality.cpp:353
|
No locals.
|
#16 0x00007ffff7deb503 in _dl_init_internal () from /lib64/ld-linux-x86-64.so.2
|
No symbol table info available.
|
#17 0x00007ffff7defb44 in dl_open_worker () from /lib64/ld-linux-x86-64.so.2
|
No symbol table info available.
|
#18 0x00007ffff7deb314 in _dl_catch_error () from /lib64/ld-linux-x86-64.so.2
|
No symbol table info available.
|
#19 0x00007ffff7def25b in _dl_open () from /lib64/ld-linux-x86-64.so.2
|
No symbol table info available.
|
#20 0x00007ffff75fc02b in dlopen_doit () from /lib64/libdl.so.2
|
No symbol table info available.
|
#21 0x00007ffff7deb314 in _dl_catch_error () from /lib64/ld-linux-x86-64.so.2
|
No symbol table info available.
|
#22 0x00007ffff75fc62d in _dlerror_run () from /lib64/libdl.so.2
|
No symbol table info available.
|
#23 0x00007ffff75fc0c1 in dlopen@@GLIBC_2.2.5 () from /lib64/libdl.so.2
|
No symbol table info available.
|
#24 0x00007fffefb6a999 in py_dl_open () at /usr/local/src/conda/python-3.7.3/Modules/_ctypes/callproc.c:1365
|