Details
-
Bug
-
Resolution: Not a Bug
-
Critical
-
Cheshire-Cat
-
Triaged
-
-
1
-
Unknown
-
KV-Engine 2021-March
Description
Build : 7.0.0-4752
Test : -test tests/integration/cheshirecat/test_cheshirecat_longevity_no_eventing.yml -scope tests/integration/cheshirecat/scope_cheshirecat_with_backup.yml (longevity without eventing)
Scale : 2
Iteration : 1st
The following KV nodes are in warmup state : 172.23.120.77, 172.23.123.24, 172.23.123.26, 172.23.96.14, 172.23.97.112, 172.23.97.241, 172.23.97.74.
On the diag logs, the following type of messages can be seen for each of the above nodes :
Control connection to memcached on 'ns_1@172.23.123.24' disconnected. Check logs for details.
|
ns_memcached 000
|
ns_1@172.23.123.24
|
3:14:52 PM 23 Mar, 2021
|
On 172.23.123.24's debug.log, the following crash report can be seen.
[error_logger:error,2021-03-23T15:14:52.453-07:00,ns_1@172.23.123.24:<0.3431.1>:ale_error_logger_handler:do_log:107]
|
=========================CRASH REPORT=========================
|
crasher:
|
initial call: erlang:apply/2
|
pid: <0.3431.1>
|
registered_name: []
|
exception error: no function clause matching
|
mc_binary:decode_header(res,
|
<<128,87,0,9,31,1,1,137,0,0,2,73,
|
38,49,1,137,22,111,24,119,107,
|
79,0,0>>) (src/mc_binary.erl, line 205)
|
in function mc_binary:quick_active_recv/3 (src/mc_binary.erl, line 55)
|
in call from mc_binary:'-get_keys_recv/5-fun-0-'/5 (src/mc_binary.erl, line 324)
|
in call from lists:foldl/3 (lists.erl, line 1263)
|
in call from mc_binary:get_keys_recv/5 (src/mc_binary.erl, line 322)
|
in call from mc_binary:do_get_keys/4 (src/mc_binary.erl, line 462)
|
in call from mc_binary:get_keys/5 (src/mc_binary.erl, line 440)
|
in call from ns_memcached:do_handle_call/3 (src/ns_memcached.erl, line 632)
|
ancestors: ['ns_memcached-bucket5',<0.3393.1>,
|
'single_bucket_kv_sup-bucket5',ns_bucket_sup,
|
ns_bucket_worker_sup,ns_server_sup,ns_server_nodes_sup,
|
<0.14793.0>,ns_server_cluster_sup,root_sup,<0.138.0>]
|
message_queue_len: 0
|
messages: []
|
links: [<0.3394.1>,#Port<0.1939>]
|
dictionary: [{sockname,{{127,0,0,1},52996}},
|
{last_call,
|
{get_keys,
|
[316,317,318,319,344,345,346,347,348,349,350,351,
|
352,353,354,355,356,357,358,359,360,361,362,363,
|
364,365,366,367,368,369,370,371,372,373,374,375,
|
376,377,378,379,380,381,382|...],
|
[{collection_uid,15},
|
{include_docs,false},
|
{inclusive_end,true},
|
{limit,100},
|
{start_key,undefined},
|
{end_key,undefined}]}}]
|
trap_exit: false
|
status: running
|
heap_size: 4185
|
stack_size: 27
|
reductions: 3079939156
|
neighbours:
|
|
|
[ns_server:debug,2021-03-23T15:14:52.453-07:00,ns_1@172.23.123.24:ns_memcached-bucket5<0.3394.1>:ns_memcached:handle_info:769]Got {'EXIT',<0.3431.1>,
|
{function_clause,
|
[{mc_binary,decode_header,
|
[res,
|
<<128,87,0,9,31,1,1,137,0,0,2,73,38,49,1,137,22,111,24,119,
|
107,79,0,0>>],
|
[{file,"src/mc_binary.erl"},{line,205}]},
|
{mc_binary,quick_active_recv,3,
|
[{file,"src/mc_binary.erl"},{line,55}]},
|
{mc_binary,'-get_keys_recv/5-fun-0-',5,
|
[{file,"src/mc_binary.erl"},{line,324}]},
|
{lists,foldl,3,[{file,"lists.erl"},{line,1263}]},
|
{mc_binary,get_keys_recv,5,
|
[{file,"src/mc_binary.erl"},{line,322}]},
|
{mc_binary,do_get_keys,4,[{file,"src/mc_binary.erl"},{line,462}]},
|
{mc_binary,get_keys,5,[{file,"src/mc_binary.erl"},{line,440}]},
|
{ns_memcached,do_handle_call,3,
|
[{file,"src/ns_memcached.erl"},{line,632}]}]}}. Exiting.
|
[user:info,2021-03-23T15:14:52.467-07:00,ns_1@172.23.123.24:ns_memcached-bucket5<0.3394.1>:ns_memcached:do_terminate:821]Control connection to memcached on 'ns_1@172.23.123.24' disconnected. Check logs for details.
|
[ns_server:debug,2021-03-23T15:14:52.468-07:00,ns_1@172.23.123.24:ns_memcached-bucket5<0.3394.1>:ns_memcached:terminate:783]Terminated.
|
[error_logger:error,2021-03-23T15:14:52.468-07:00,ns_1@172.23.123.24:ns_memcached-bucket5<0.3394.1>:ale_error_logger_handler:do_log:107]
|
=========================ERROR REPORT=========================
|
** Generic server 'ns_memcached-bucket5' terminating
|
** Last message in was {'EXIT',<0.3431.1>,
|
{function_clause,
|
[{mc_binary,decode_header,
|
[res,
|
<<128,87,0,9,31,1,1,137,0,0,2,73,38,49,1,
|
137,22,111,24,119,107,79,0,0>>],
|
[{file,"src/mc_binary.erl"},{line,205}]},
|
{mc_binary,quick_active_recv,3,
|
[{file,"src/mc_binary.erl"},{line,55}]},
|
{mc_binary,'-get_keys_recv/5-fun-0-',5,
|
[{file,"src/mc_binary.erl"},{line,324}]},
|
{lists,foldl,3,
|
[{file,"lists.erl"},{line,1263}]},
|
{mc_binary,get_keys_recv,5,
|
[{file,"src/mc_binary.erl"},{line,322}]},
|
{mc_binary,do_get_keys,4,
|
[{file,"src/mc_binary.erl"},{line,462}]},
|
{mc_binary,get_keys,5,
|
[{file,"src/mc_binary.erl"},{line,440}]},
|
{ns_memcached,do_handle_call,3,
|
[{file,"src/ns_memcached.erl"},
|
{line,632}]}]}}
|
** When Server state == {state,0,1,0,
|
{[],[]},
|
{[],[]},
|
{[],[]},
|
warmed,
|
{1616,456819,721399},
|
"bucket5",
|
[collections,json],
|
#Port<0.1932>,
|
[{<0.3433.1>,
|
#Ref<0.1353251921.3048472582.147525>},
|
{<0.3434.1>,
|
#Ref<0.1353251921.3038511112.7496>},
|
{<0.3432.1>,
|
#Ref<0.1353251921.3040083971.255261>}],
|
[],undefined}
|
** Reason for termination ==
|
** {function_clause,[{mc_binary,decode_header,
|
[res,
|
<<128,87,0,9,31,1,1,137,0,0,2,73,38,49,1,137,
|
22,111,24,119,107,79,0,0>>],
|
[{file,"src/mc_binary.erl"},{line,205}]},
|
{mc_binary,quick_active_recv,3,
|
[{file,"src/mc_binary.erl"},{line,55}]},
|
{mc_binary,'-get_keys_recv/5-fun-0-',5,
|
[{file,"src/mc_binary.erl"},{line,324}]},
|
{lists,foldl,3,[{file,"lists.erl"},{line,1263}]},
|
{mc_binary,get_keys_recv,5,
|
[{file,"src/mc_binary.erl"},{line,322}]},
|
{mc_binary,do_get_keys,4,
|
[{file,"src/mc_binary.erl"},{line,462}]},
|
{mc_binary,get_keys,5,
|
[{file,"src/mc_binary.erl"},{line,440}]},
|
{ns_memcached,do_handle_call,3,
|
[{file,"src/ns_memcached.erl"},
|
{line,632}]}]}
|
|
|
[error_logger:error,2021-03-23T15:14:52.478-07:00,ns_1@172.23.123.24:ns_memcached-bucket5<0.3394.1>:ale_error_logger_handler:do_log:107]
|
=========================CRASH REPORT=========================
|
crasher:
|
initial call: ns_memcached:init/1
|
pid: <0.3394.1>
|
registered_name: 'ns_memcached-bucket5'
|
exception exit: {function_clause,
|
[{mc_binary,decode_header,
|
[res,
|
<<128,87,0,9,31,1,1,137,0,0,2,73,38,49,1,137,22,
|
111,24,119,107,79,0,0>>],
|
[{file,"src/mc_binary.erl"},{line,205}]},
|
{mc_binary,quick_active_recv,3,
|
[{file,"src/mc_binary.erl"},{line,55}]},
|
{mc_binary,'-get_keys_recv/5-fun-0-',5,
|
[{file,"src/mc_binary.erl"},{line,324}]},
|
{lists,foldl,3,[{file,"lists.erl"},{line,1263}]},
|
{mc_binary,get_keys_recv,5,
|
[{file,"src/mc_binary.erl"},{line,322}]},
|
{mc_binary,do_get_keys,4,
|
[{file,"src/mc_binary.erl"},{line,462}]},
|
{mc_binary,get_keys,5,
|
[{file,"src/mc_binary.erl"},{line,440}]},
|
{ns_memcached,do_handle_call,3,
|
[{file,"src/ns_memcached.erl"},{line,632}]}]}
|
in function gen_server:handle_common_reply/8 (gen_server.erl, line 751)
|
ancestors: [<0.3393.1>,'single_bucket_kv_sup-bucket5',ns_bucket_sup,
|
ns_bucket_worker_sup,ns_server_sup,ns_server_nodes_sup,
|
<0.14793.0>,ns_server_cluster_sup,root_sup,<0.138.0>]
|
message_queue_len: 0
|
messages: []
|
links: [<0.3433.1>,<0.3435.1>,#Port<0.1932>,<0.3436.1>,<0.3434.1>,
|
<0.3432.1>,<0.3393.1>]
|
dictionary: []
|
trap_exit: true
|
status: running
|
heap_size: 514838
|
stack_size: 27
|
reductions: 19541903
|
neighbours:
|
Similar crash reports are seen on all of the above nodes.
Attachments
Issue Links
- relates to
-
-
- Closed
-