Details
-
Bug
-
Resolution: Fixed
-
Critical
-
Cheshire-Cat
-
7.0.0-3594
-
Triaged
-
Centos 64-bit
-
1
-
Unknown
Description
Steps to repro:*
- Created a 4 node cluster (replicas =1)
- Loaded 10 million items(doc size = 256)
- Started New doc ops (create:expiry).
- During doc ops restarting couchbase(graceful restart)
- Observed Memcached crashed in cb::rbac::PrivilegeDatabase::lookup (this=this@entry=0x0, user=...)
crash is seen on 172.23.107.81
BackTrace:
(gdb) bt full
|
#0 0x0000000000577adf in hash (__seed=3339675911, __clength=<optimized out>, __ptr=<optimized out>)
|
at /usr/local/include/c++/7.3.0/bits/functional_hash.h:192
|
No locals.
|
#1 operator() (this=0x8, __s=...) at /usr/local/include/c++/7.3.0/bits/basic_string.h:6591
|
No locals.
|
#2 _M_hash_code (this=0x8, __k=...) at /usr/local/include/c++/7.3.0/bits/hashtable_policy.h:1368
|
No locals.
|
#3 find (__k=..., this=0x8) at /usr/local/include/c++/7.3.0/bits/hashtable.h:1436
|
__code = <optimized out>
|
__n = <optimized out>
|
#4 find (__x=..., this=0x8) at /usr/local/include/c++/7.3.0/bits/unordered_map.h:927
|
No locals.
|
#5 cb::rbac::PrivilegeDatabase::lookup (this=this@entry=0x0, user=...)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/rbac/privilege_database.cc:419
|
iter = <optimized out>
|
#6 0x0000000000578635 in cb::rbac::PrivilegeDatabase::createContext (this=0x0, user=..., domain=<optimized out>, bucket=...)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/rbac/privilege_database.cc:383
|
ue = <optimized out>
|
iter = <optimized out>
|
#7 0x000000000057b671 in cb::rbac::createContext (user=..., bucket=...)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/rbac/privilege_database.cc:516
|
ctx = <optimized out>
|
#8 0x0000000000442a27 in prometheus_auth_callback (user=..., password=...)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/memcached.cc:246
|
No locals.
|
#9 0x000000000054479d in operator() (__args#1=..., __args#0=..., this=0x7f19da2480e8)
|
at /usr/local/include/c++/7.3.0/bits/std_function.h:706
|
No locals.
|
#10 prometheus::BasicAuthHandler::AuthorizeInner (this=this@entry=0x7f19da2480e0, conn=conn@entry=0x1a19210)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/pull/src/basic_auth.cc:64
|
authHeader = <optimized out>
|
authHeaderStr = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
|
_M_p = 0x7f19d95d0d80 <Address 0x7f19d95d0d80 out of bounds>}, _M_string_length = 366, {
|
_M_local_buf = "n\001\000\000\000\000\000\000\204&T\000\000\000\000", _M_allocated_capacity = 366}}
|
---Type <return> to continue, or q <return> to quit---
|
prefix = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
|
_M_p = 0x7f19d5cdaa40 "Basic "}, _M_string_length = 6, {_M_local_buf = "Basic \000\000=zs\334\031\177\000",
|
_M_allocated_capacity = 35611343020354}}
|
b64Auth = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
|
_M_p = 0x7f19d95d0f00 <Address 0x7f19d95d0f00 out of bounds>}, _M_string_length = 360, {
|
_M_local_buf = "h\001", '\000' <repeats 13 times>, _M_allocated_capacity = 360}}
|
decoded = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
|
_M_p = 0x7f19d956fdc0 <Address 0x7f19d956fdc0 out of bounds>}, _M_string_length = 268, {
|
_M_local_buf = "\016\001\000\000\000\000\000\000\002\000\000\000\000\000\000", _M_allocated_capacity = 270}}
|
splitPos = 11
|
username = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
|
_M_p = 0x7f19d5cdaac0 "@prometheus"}, _M_string_length = 11, {_M_local_buf = "@prometheus\000\031\177\000",
|
_M_allocated_capacity = 7526752397938552896}}
|
password = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
|
_M_p = 0x7f19d956f640 <Address 0x7f19d956f640 out of bounds>}, _M_string_length = 256, {
|
_M_local_buf = "\000\001\000\000\000\000\000\000\016_W\000\000\000\000", _M_allocated_capacity = 256}}
|
#11 0x0000000000544c01 in prometheus::BasicAuthHandler::authorize (this=0x7f19da2480e0, server=<optimized out>, conn=0x1a19210)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/pull/src/basic_auth.cc:16
|
No locals.
|
#12 0x0000000000531f19 in CivetServer::authHandler (conn=<optimized out>, cbdata=0x7f19da2480e0)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/CivetServer.cpp:179
|
request_info = <optimized out>
|
me = 0x7f19da247fa0
|
handler = 0x7f19da2480e0
|
#13 0x000000000053c762 in handle_request (conn=conn@entry=0x1a19210)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:14203
|
ri = 0x1a19218
|
path = '\000' <repeats 4095 times>
|
---Type <return> to continue, or q <return> to quit---
|
uri_len = <optimized out>
|
is_found = 0
|
is_script_resource = 1
|
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
|
is_put_or_delete_request = 0
|
is_callback_resource = 1
|
i = <optimized out>
|
file = {stat = {size = 0, last_modified = 0, is_directory = 0, is_gzipped = 0, location = 0}, access = {fp = 0x0}}
|
callback_handler = 0x531c50 <CivetServer::requestHandler(mg_connection*, void*)>
|
handler_info = 0x1a1af90
|
subprotocols = 0x0
|
ws_connect_handler = 0x0
|
ws_ready_handler = 0x0
|
ws_data_handler = 0x0
|
ws_close_handler = 0x0
|
callback_data = 0x7f19da230000
|
auth_handler = 0x531e60 <CivetServer::authHandler(mg_connection*, void*)>
|
auth_callback_data = 0x7f19da2480e0
|
handler_type = 0
|
curtime = 1604249442
|
date = '\000' <repeats 63 times>
|
__func__ = "handle_request"
|
#14 0x000000000053d7b4 in process_new_connection (conn=0x1a19210)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:18048
|
ebuf = '\000' <repeats 99 times>
|
ri = 0x1a19218
|
discard_len = <optimized out>
|
reqerr = 0
|
#15 worker_thread_run (conn=0x1a19210)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:18430
|
ctx = 0x1a18710
|
thread_index = <optimized out>
|
tls = {is_master = 0, thread_idx = 2, user_ptr = 0x0}
|
#16 worker_thread (thread_func_param=0x1a19210)
|
---Type <return> to continue, or q <return> to quit---
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:18486
|
sa = {__sigaction_handler = {sa_handler = 0x1, sa_sigaction = 0x1}, sa_mask = {__val = {0 <repeats 16 times>}}, sa_flags = 0,
|
sa_restorer = 0x0}
|
#17 0x00007f19dac77e65 in start_thread () from /lib64/libpthread.so.0
|
No symbol table info available.
|
#18 0x00007f19da9a088d in clone () from /lib64/libc.so.6
|
No symbol table info available.
|
(gdb)
|
QE test:
./testrunner -i /tmp/testexec.4987.ini bucket_storage=couchstore,rerun=false,GROUP=P0;long_graceful,randomize_value=true,doc_size=256,bucket_eviction_policy=fullEviction,replicas=1,nodes_init=4,upgrade_version=7.0.0-3594 -t magma.magma_crash_recovery.MagmaCrashTests.test_crash_during_ops,num_items=1000000,doc_size=256,sdk_timeout=60,graceful=True,doc_ops=create:expiry,replicas=1,GROUP=P0;long_graceful,multiplier=100
|
Attachments
Issue Links
- is duplicated by
-
MB-43016 memcached crash observed in build sanity
- Closed