Uploaded image for project: 'Couchbase Server'
  1. Couchbase Server
  2. MB-11966

Unexpected message during Auto failover

    XMLWordPrintable

Details

    • Bug
    • Resolution: Won't Fix
    • Major
    • 3.0
    • 3.0
    • ns_server
    • Security Level: Public
    • None
    • Untriaged
    • Unknown

    Description

      Unexpected message during Auto failover

      Node ('ns_1@10.6.2.171') was automatically failovered.
      [down,stale,
      {last_heard,{1408,61265,304895}},
      {now,{1408,61265,292555}},

      {active_buckets,["default","saslbucket"]}

      ,

      {ready_buckets,["default","saslbucket"]}

      ,

      {status_latency,14467}

      ,
      {outgoing_replications_safeness_level,
      [

      {"saslbucket",green}

      ,

      {"default",green}

      ]},
      {incoming_replications_conf_hashes,
      [{"saslbucket",
      [

      {'ns_1@10.6.2.164',70535071}, {'ns_1@10.6.2.165',61731168}, {'ns_1@10.6.2.166',118079904}, {'ns_1@10.6.2.167',82759527}, {'ns_1@10.6.2.168',66162842}, {'ns_1@10.6.2.172',80119058}, {'ns_1@10.6.2.187',11556200}]},
      {"default",
      [{'ns_1@10.6.2.164',70535071}

      ,

      {'ns_1@10.6.2.165',61731168}

      ,

      {'ns_1@10.6.2.166',118079904}

      ,

      {'ns_1@10.6.2.167',82759527}

      ,

      {'ns_1@10.6.2.168',66162842}

      ,

      {'ns_1@10.6.2.172',80119058}

      ,

      {'ns_1@10.6.2.187',11556200}

      ]}]},
      {local_tasks,
      [[

      {pid,<<"<0.6849.11>">>}

      ,

      {changes_done,42549}

      ,

      {design_documents,[<<"_design/ddoc1">>]}

      ,

      {indexer_type,main}

      ,

      {initial_build,true}

      ,

      {progress,6}

      ,

      {set,<<"default">>}

      ,

      {signature,<<"b7af3238f0f1cba7b76df8a22bd892a8">>}

      ,

      {started_on,1408061255}

      ,

      {total_changes,630488}

      ,

      {type,indexer}

      ,

      {updated_on,1408061260}

      ]]},
      {memory,
      [

      {total,193455704}

      ,

      {processes,141599192}

      ,

      {processes_used,140731328}

      ,

      {system,51856512}

      ,

      {atom,635505}

      ,

      {atom_used,605334}

      ,

      {binary,17515856}

      ,

      {code,15856645}

      ,

      {ets,9199168}

      ]},
      {system_memory_data,
      [

      {system_total_memory,24502919168}

      ,

      {free_swap,11089731584}

      ,

      {total_swap,11089731584}

      ,

      {cached_memory,2014679040}

      ,

      {buffered_memory,276660224}

      ,

      {free_memory,20389609472}

      ,

      {total_memory,24502919168}

      ]},
      {node_storage_conf,[

      {db_path,"/data"}

      ,

      {index_path,"/index"}

      ]},
      {statistics,
      [{wall_clock,{3272953,5006}},
      {context_switches,{50781753,0}},
      {garbage_collection,{7761032,40527599965,0}},
      {io,{{input,14508418035},

      {output,5090410762}

      }},
      {reductions,{15933314301,20988615}},

      {run_queue,0}

      ,
      {runtime,{4135530,8890}},
      {run_queues,{0,0,0,0,0,0,0,0}}]},
      {system_stats,
      [

      {cpu_utilization_rate,26.73392181588903}

      ,

      {swap_total,11089731584}

      ,

      {swap_used,0}

      ,

      {mem_total,24502919168}

      ,

      {mem_free,22689263616}

      ]},
      {interesting_stats,
      [

      {cmd_get,0.0}, {couch_docs_actual_disk_size,816975640}, {couch_docs_data_size,701174710}, {couch_views_actual_disk_size,58784529}, {couch_views_data_size,21468931}, {curr_items,1220054}, {curr_items_tot,2440044}, {ep_bg_fetched,0.0}, {get_hits,0.0}, {mem_used,842905312}, {ops,0.0}, {vb_replica_curr_items,1219990}]},
      {per_bucket_interesting_stats,
      [{"default",
      [{cmd_get,0.0}

      ,

      {couch_docs_actual_disk_size,392123814}

      ,

      {couch_docs_data_size,360133448}

      ,

      {couch_views_actual_disk_size,6267953}

      ,

      {couch_views_data_size,9463}

      ,

      {curr_items,630488}

      ,

      {curr_items_tot,1260556}

      ,

      {ep_bg_fetched,0.0}, {get_hits,0.0}, {mem_used,434981936}, {ops,0.0}, {vb_replica_curr_items,630068}]},
      {"saslbucket",
      [{cmd_get,0.0}, {couch_docs_actual_disk_size,424851826}, {couch_docs_data_size,341041262}, {couch_views_actual_disk_size,52516576}, {couch_views_data_size,21459468}, {curr_items,589566}, {curr_items_tot,1179488},{ep_bg_fetched,0.0}

      ,

      {get_hits,0.0}

      ,

      {mem_used,407923376}

      ,

      {ops,0.0}

      ,

      {vb_replica_curr_items,589922}

      ]}]},
      {processes_stats,
      [

      {<<"proc/(main)beam.smp/cpu_utilization">>,0}

      ,

      {<<"proc/(main)beam.smp/major_faults">>,0}

      ,

      {<<"proc/(main)beam.smp/major_faults_raw">>,0}

      ,

      {<<"proc/(main)beam.smp/mem_resident">>,403845120}

      ,

      {<<"proc/(main)beam.smp/mem_share">>,10874880}

      ,

      {<<"proc/(main)beam.smp/mem_size">>,3159314432}

      ,

      {<<"proc/(main)beam.smp/minor_faults">>,7718}

      ,

      {<<"proc/(main)beam.smp/minor_faults_raw">>,14516584}

      ,

      {<<"proc/(main)beam.smp/page_faults">>,7718}

      ,

      {<<"proc/(main)beam.smp/page_faults_raw">>,14516584}

      ,

      {<<"proc/beam.smp/cpu_utilization">>,0}

      ,

      {<<"proc/beam.smp/major_faults">>,0}

      ,

      {<<"proc/beam.smp/major_faults_raw">>,0}

      ,

      {<<"proc/beam.smp/mem_resident">>,28205056}

      ,

      {<<"proc/beam.smp/mem_share">>,3162112}

      ,

      {<<"proc/beam.smp/mem_size">>,938668032}

      ,

      {<<"proc/beam.smp/minor_faults">>,0}

      ,

      {<<"proc/beam.smp/minor_faults_raw">>,8604}

      ,

      {<<"proc/beam.smp/page_faults">>,0}

      ,

      {<<"proc/beam.smp/page_faults_raw">>,8604}

      ,

      {<<"proc/inet_gethost/cpu_utilization">>,206}

      ,

      {<<"proc/inet_gethost/major_faults">>,0}

      ,

      {<<"proc/inet_gethost/major_faults_raw">>,0}

      ,

      {<<"proc/inet_gethost/mem_resident">>,544768}

      ,

      {<<"proc/inet_gethost/mem_share">>,438272}

      ,

      {<<"proc/inet_gethost/mem_size">>,11051008}

      ,

      {<<"proc/inet_gethost/minor_faults">>,0}

      ,

      {<<"proc/inet_gethost/minor_faults_raw">>,615}

      ,

      {<<"proc/inet_gethost/page_faults">>,0}

      ,

      {<<"proc/inet_gethost/page_faults_raw">>,615}

      ,

      {<<"proc/memcached/cpu_utilization">>,0}

      ,

      {<<"proc/memcached/major_faults">>,0}

      ,

      {<<"proc/memcached/major_faults_raw">>,0}

      ,

      {<<"proc/memcached/mem_resident">>,956690432}

      ,

      {<<"proc/memcached/mem_share">>,7032832}

      ,

      {<<"proc/memcached/mem_size">>,1293410304}

      ,

      {<<"proc/memcached/minor_faults">>,0}

      ,

      {<<"proc/memcached/minor_faults_raw">>,244699}

      ,

      {<<"proc/memcached/page_faults">>,0}

      ,

      {<<"proc/memcached/page_faults_raw">>,244699}

      ]},

      {cluster_compatibility_version,196608}

      ,
      {version,
      [

      {lhttpc,"1.3.0"}

      ,

      {os_mon,"2.2.14"}

      ,

      {public_key,"0.21"}

      ,

      {asn1,"2.0.4"}

      ,

      {couch,"2.1.1r-425-g6e19fa4"}

      ,

      {kernel,"2.16.4"}

      ,

      {syntax_tools,"1.6.13"}

      ,

      {xmerl,"1.3.6"}

      ,

      {ale,"3.0.0-1156-rel-enterprise"}

      ,

      {couch_set_view,"2.1.1r-425-g6e19fa4"}

      ,

      {compiler,"4.9.4"}

      ,

      {inets,"5.9.8"}

      ,

      {mapreduce,"1.0.0"}

      ,

      {couch_index_merger,"2.1.1r-425-g6e19fa4"}

      ,

      {ns_server,"3.0.0-1156-rel-enterprise"}

      ,

      {oauth,"7d85d3ef"}

      ,

      {crypto,"3.2"}

      ,

      {ssl,"5.3.3"}

      ,

      {sasl,"2.3.4"}

      ,

      {couch_view_parser,"1.0.0"}

      ,

      {mochiweb,"2.4.2"}

      ,

      {stdlib,"1.19.4"}

      ]},

      {supported_compat_version,[3,0]}

      ,

      {advertised_version,[3,0,0]}

      ,

      {system_arch,"x86_64-unknown-linux-gnu"}

      ,

      {wall_clock,3272}

      ,
      {memory_data,{24502919168,4118437888,

      {<17969.16131.9>,8238288}

      }},
      {disk_data,
      [

      {"/",35277560,7}

      ,

      {"/dev/shm",11964316,0}

      ,

      {"/boot",495844,7}

      ,

      {"/data",258026884,1}

      ,

      {"/index",175456944,1}

      ]},

      {meminfo, <<"MemTotal: 23928632 kB\nMemFree: 19911728 kB\nBuffers: 270176 kB\nCached: 1967460 kB\nSwapCached: 0 kB\nActive: 2088040 kB\nInactive: 1565640 kB\nActive(anon): 1376536 kB\nInactive(anon): 39796 kB\nActive(file): 711504 kB\nInactive(file): 1525844 kB\nUnevictable: 0 kB\nMlocked: 0 kB\nSwapTotal: 10829816 kB\nSwapFree: 10829816 kB\nDirty: 13160 kB\nWriteback: 0 kB\nAnonPages: 1416624 kB\nMapped: 15856 kB\nShmem: 144 kB\nSlab: 171188 kB\nSReclaimable: 111136 kB\nSUnreclaim: 60052 kB\nKernelStack: 2664 kB\nPageTables: 6560 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 22794132 kB\nCommitted_AS: 2378568 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 66532 kB\nVmallocChunk: 34359541352 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 0 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 24576000 kB\nDirectMap2M: 0 kB\n">>}

      ]

      Will attach logs

      Attachments

        No reviews matched the request. Check your Options in the drop-down menu of this sections header.

        Activity

          People

            alkondratenko Aleksey Kondratenko (Inactive)
            parag Parag Agarwal (Inactive)
            Votes:
            0 Vote for this issue
            Watchers:
            2 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved:

              Gerrit Reviews

                There are no open Gerrit changes

                PagerDuty