Uploaded image for project: 'Couchbase Server'
  1. Couchbase Server
  2. MB-13069

Rebalance failed due to Bad Replicas during Delta Recovery

    XMLWordPrintable

Details

    • Bug
    • Resolution: Fixed
    • Blocker
    • 4.0.0
    • 4.0.0
    • ns_server
    • Security Level: Public
    • None
    • 1:10.3.5.115
      2:10.3.5.116
      3:10.3.5.117
      4:10.3.5.118
      5:10.6.2.185
      6:10.6.2.186

      Build Version:: 3.5.0-861

    Description

      Test Case::
      ./testrunner i centos_x6401_01-failover_upr.ini get-cbcollect-info=False,get-logs=False,stop-on-failure=False,get-coredumps=False,force_kill_memached=True,std_vbucket_dist=20,total_vbuckets=128 -t failover.failovertests.FailoverTests.test_failover_then_add_back,skip_cleanup=True,replicas=1,num_failed_nodes=1,items=100000,standard_buckets=1,recoveryType=delta,graceful=True,GROUP=P2

      SCENARIO

      1. Create a 6 node cluster
      2. Add 1 Standard Bucket, 1 Default and load each with 100000 items
      3. Failvoer 1 Node and do delta recovery
      4. Rebalance node

      Error Observed: Rebalance failed due to Bad Replicas

      2015-01-09 17:34:24 | ERROR | MainProcess | test_thread | [rest_client.print_UI_logs] {u'node': u'ns_1@10.3.5.115', u'code': 2, u'text': u"Bad replicators after rebalance:\nMissing = [

      {'ns_1@10.3.5.116','ns_1@10.6.2.185',279}

      ,\n

      {'ns_1@10.3.5.116','ns_1@10.6.2.185',282}

      ,\n

      {'ns_1@10.3.5.116','ns_1@10.6.2.185',283}

      ,\n

      {'ns_1@10.3.5.116','ns_1@10.6.2.185',284}

      ,\n

      {'ns_1@10.3.5.116','ns_1@10.6.2.185',286}

      ,\n

      {'ns_1@10.3.5.116','ns_1@10.6.2.185',287}

      ,\n

      {'ns_1@10.3.5.116','ns_1@10.6.2.185',288}

      ,\n

      {'ns_1@10.3.5.116','ns_1@10.6.2.185',289}

      ,\n

      {'ns_1@10.3.5.116','ns_1@10.6.2.185',290}

      ,\n

      {'ns_1@10.3.5.116','ns_1@10.6.2.185',291}

      ,\n

      {'ns_1@10.3.5.116','ns_1@10.6.2.185',292}

      ,\n

      {'ns_1@10.3.5.116','ns_1@10.6.2.185',293}

      ,\n

      {'ns_1@10.3.5.116','ns_1@10.6.2.185',294}

      ,\n

      {'ns_1@10.3.5.116','ns_1@10.6.2.185',295}

      ,\n

      {'ns_1@10.3.5.116','ns_1@10.6.2.185',296}

      ,\n

      {'ns_1@10.3.5.116','ns_1@10.6.2.185',297}

      ,\n

      {'ns_1@10.3.5.116','ns_1@10.6.2.185',298}

      ,\n

      {'ns_1@10.3.5.116','ns_1@10.6.2.185',299}

      ,\n

      {'ns_1@10.3.5.116','ns_1@10.6.2.185',300}

      ,\n

      {'ns_1@10.3.5.116','ns_1@10.6.2.185',301}

      ,\n

      {'ns_1@10.3.5.116','ns_1@10.6.2.185',302}

      ,\n

      {'ns_1@10.3.5.116','ns_1@10.6.2.185',303}

      ,\n

      {'ns_1@10.3.5.116','ns_1@10.6.2.185',304}

      ,\n

      {'ns_1@10.3.5.116','ns_1@10.6.2.185',305}

      ,\n

      {'ns_1@10.3.5.116','ns_1@10.6.2.185',306}

      ,\n

      {'ns_1@10.3.5.116','ns_1@10.6.2.185',307}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',445}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',446}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',447}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',448}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',449}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',450}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',451}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',452}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',453}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',454}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',455}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',456}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',457}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',458}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',459}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',460}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',461}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',462}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',463}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',464}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',465}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',466}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',467}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',468}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',469}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',470}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',471}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',472}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',473}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',474}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',475}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',476}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',477}

      ,\n

      {'ns_1@10.3.5.117','ns_1@10.6.2.185',478}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',616}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',617}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',618}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',619}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',620}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',621}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',622}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',623}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',624}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',625}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',626}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',627}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',628}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',629}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',630}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',631}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',632}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',633}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',634}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',635}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',636}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',637}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',638}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',639}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',640}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',641}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',642}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',643}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',644}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',645}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',646}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',647}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',648}

      ,\n

      {'ns_1@10.3.5.118','ns_1@10.6.2.185',649}

      Attachments

        For Gerrit Dashboard: MB-13069
        # Subject Branch Project Status CR V

        Activity

          People

            mikew Mike Wiederhold [X] (Inactive)
            parag Parag Agarwal (Inactive)
            Votes:
            0 Vote for this issue
            Watchers:
            2 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved:

              Gerrit Reviews

                There are no open Gerrit changes

                PagerDuty