Details
-
Bug
-
Resolution: Fixed
-
Critical
-
Cheshire-Cat
-
Enterprise Edition 7.0.0 build 4214
-
Untriaged
-
Centos 64-bit
-
-
1
-
Unknown
Description
Build: 7.0.0-4214
Scenario:
- 9 node cluster (3 kv + 4 fts + 2 n1ql,2i)
- 6 couchstore buckets with replica=1
- Create 50 GSI on each bucket
- Create 10 fts index with index_partition=6
- Create and drop 100 GSIs on 4 buckets (total 400 indexes)
- Create and drop 50 FTS indexes on 3 buckets (Total 150 fts)
- Rebalance out one KV node
Observation:
Rebalance_out of KV node failed with reason "failed to update vbucket map" shortly followed by ns_server crash with exit status 137
Worker <0.18328.59> (for action {move,{810,
|
['ns_1@172.23.105.212', 'ns_1@172.23.105.155'],
|
['ns_1@172.23.105.155', 'ns_1@172.23.105.211'],
|
[]}}) exited with reason {unexpected_exit,
|
{'EXIT', <0.18901.59>,
|
{failed_to_update_vbucket_map, "couchstore-5", 810, {error,
|
[{'ns_1@172.23.105.213',
|
{exit, {shutdown,
|
{gen_server, call,
|
[{ns_config_rep, 'ns_1@172.23.105.213'},
|
synchronize_everything, infinity]}}}}]}}}} Rebalance exited with reason {mover_crashed,
|
{unexpected_exit, {'EXIT',<0.18901.59>,
|
{failed_to_update_vbucket_map,"couchstore-5", 810, {error, [{'ns_1@172.23.105.213',
|
{exit, {shutdown, {gen_server,call,
|
[{ns_config_rep,'ns_1@172.23.105.213'}, synchronize_everything, infinity]}}}}]}}}}}.
|
Rebalance Operation Id = 6d62fcfdc0b46b78ab89ad8c1467999fService 'ns_server' exited with status 137. Restarting. Messages:
|
80c860daa3d6b8bac8f03ed-1610714375118/couchstore-1_fts_1">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{1,63777933575}},{<<"c51b5923865d4c2ce90e622f17bf8e60">>,{1,63777933669}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c0e484f080c860daa3d6b8bac8f03ed-1610714375118/couchstore-1_fts_2">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{1,63777933576}},{<<"c51b5923865d4c2ce90e622f17bf8e60">>,{1,63777933669}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c0e484f080c860daa3d6b8bac8f03ed-1610714375118/couchstore-1_fts_3">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{1,63777933575}},{<<"c51b5923865d4c2ce90e622f17bf8e60">>,{1,63777933669}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c0e484f080c860daa3d6b8bac8f03ed-1610714375118/couchstore-1_fts_4">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{1,63777933575}},{<<"c51b5923865d4c2ce90e622f17bf8e60">>,{1,63777933669}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c0e484f080c860daa3d6b8bac8f03ed-1610714375118/couchstore-1_fts_5">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{1,63777933575}},{<<"c51b5923865d4c2ce90e622f17bf8e60">>,{1,63777933669}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c0e484f080c860daa3d6b8bac8f03ed-1610714375118/couchstore-1_fts_6">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{1,63777933575}},{<<"c51b5923865d4c2ce90e622f17bf8e60">>,{1,63777933669}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c0e484f080c860daa3d6b8bac8f03ed-1610714375118/couchstore-1_fts_7">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{1,63777933575}},{<<"c51b5923865d4c2ce90e622f17bf8e60">>,{1,63777933669}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c0e484f080c860daa3d6b8bac8f03ed-1610714375118/couchstore-1_fts_8">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{1,63777933576}},{<<"c51b5923865d4c2ce90e622f17bf8e60">>,{1,63777933669}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c0e484f080c860daa3d6b8bac8f03ed-1610714375118/couchstore-1_fts_9">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{1,63777933575}},{<<"c51b5923865d4c2ce90e622f17bf8e60">>,{1,63777933669}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c0e484f080c860daa3d6b8bac8f03ed-1610714375118/couchstore-4_fts_0">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{1,63777933575}},{<<"c51b5923865d4c2ce90e622f17bf8e60">>,{1,63777933669}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c0e484f080c860daa3d6b8bac8f03ed-1610714375118/couchstore-4_fts_1">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{1,63777933575}},{<<"c51b5923865d4c2ce90e622f17bf8e60">>,{1,63777933669}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c0e484f080c860daa3d6b8bac8f03ed-1610714375118/couchstore-4_fts_2">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{1,63777933576}},{<<"c51b5923865d4c2ce90e622f17bf8e60">>,{1,63777933669}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c0e484f080c860daa3d6b8bac8f03ed-1610714375118/couchstore-4_fts_3">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{1,63777933575}},{<<"c51b5923865d4c2ce90e622f17bf8e60">>,{1,63777933669}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c0e484f080c860daa3d6b8bac8f03ed-1610714375118/couchstore-4_fts_4">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{1,63777933575}},{<<"c51b5923865d4c2ce90e622f17bf8e60">>,{1,63777933669}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c0e484f080c860daa3d6b8bac8f03ed-1610714375118/couchstore-4_fts_5">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{1,63777933575}},{<<"c51b5923865d4c2ce90e622f17bf8e60">>,{1,63777933669}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c0e484f080c860daa3d6b8bac8f03ed-1610714375118/couchstore-4_fts_6">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{1,63777933575}},{<<"c51b5923865d4c2ce90e622f17bf8e60">>,{1,63777933669}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c0e484f080c860daa3d6b8bac8f03ed-1610714375118/couchstore-4_fts_7">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{1,63777933575}},{<<"c51b5923865d4c2ce90e622f17bf8e60">>,{1,63777933669}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c0e484f080c860daa3d6b8bac8f03ed-1610714375118/couchstore-4_fts_8">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{1,63777933576}},{<<"c51b5923865d4c2ce90e622f17bf8e60">>,{1,63777933669}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c0e484f080c860daa3d6b8bac8f03ed-1610714375118/couchstore-4_fts_9">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{1,63777933575}},{<<"c51b5923865d4c2ce90e622f17bf8e60">>,{1,63777933669}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c7e64d93b50b29ff703817dd72f03a6-1610713330909/couchstore-0_fts_0">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{2,63777932628}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c7e64d93b50b29ff703817dd72f03a6-1610713330909/couchstore-0_fts_1">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{2,63777932628}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c7e64d93b50b29ff703817dd72f03a6-1610713330909/couchstore-0_fts_2">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{2,63777932628}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c7e64d93b50b29ff703817dd72f03a6-1610713330909/couchstore-0_fts_3">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{2,63777932628}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c7e64d93b50b29ff703817dd72f03a6-1610713330909/couchstore-0_fts_4">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{2,63777932628}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c7e64d93b50b29ff703817dd72f03a6-1610713330909/couchstore-0_fts_5">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{2,63777932628}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c7e64d93b50b29ff703817dd72f03a6-1610713330909/couchstore-0_fts_6">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{2,63777932628}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c7e64d93b50b29ff703817dd72f03a6-1610713330909/couchstore-0_fts_7">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{2,63777932628}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c7e64d93b50b29ff703817dd72f03a6-1610713330909/couchstore-0_fts_8">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{2,63777932628}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c7e64d93b50b29ff703817dd72f03a6-1610713330909/couchstore-0_fts_9">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{2,63777932628}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c7e64d93b50b29ff703817dd72f03a6-1610713330909/couchstore-1_fts_0">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{2,63777932628}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c7e64d93b50b29ff703817dd72f03a6-1610713330909/couchstore-1_fts_1">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{2,63777932628}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c7e64d93b50b29ff703817dd72f03a6-1610713330909/couchstore-1_fts_2">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{2,63777932628}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c7e64d93b50b29ff703817dd72f03a6-1610713330909/couchstore-1_fts_3">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{2,63777932628}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c7e64d93b50b29ff703817
|
dd72f03a6-1610713330909/couchstore-1_fts_4">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{2,63777932628}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c7e64d93b50b29ff703817dd72f03a6-1610713330909/couchstore-1_fts_5">>},[{'_vclock',[{<<"9199a9abbd945369d99a381242bcfc6d">>,{2,63777932628}}]}|'_deleted']},{{metakv,<<"/fts/cbgt/cfg/planPIndexesLean/planPIndexesLean-2c7e64d93b50b29ff703817dd72f03a6-1610713330909/couchstore-1_fts_6">>},[{'_vclock',[{[os_mon] memory supervisor port (memsup): Erlang has closed
|
[os_mon] cpu supervisor port (cpu_sup): Erlang has closed
|
TAF test case:
rebalance_new.rebalance_with_services.ServiceRebalanceTests:
|
test_MB_43291,nodes_init=9,services_init=kv-kv-kv-index:n1ql-index:n1ql-fts-fts-fts-fts,standard_buckets=6,replicas=1,num_items=5000,num_gsi_index=50,num_fts_index=10,gsi_index_create_drop=100,fts_index_create_drop=50,fts_index_partition=6,cluster_action=rebalance_out;rebalance_in,target_nodes=kv,num_nodes_to_run=5'
|
