Uploaded image for project: 'Couchbase Server'
  1. Couchbase Server
  2. MB-8231

Rebalance hangs with progress zero percent

    XMLWordPrintable

Details

    • Bug
    • Resolution: Fixed
    • Blocker
    • 2.1.0
    • 2.1.0
    • couchbase-bucket, ns_server
    • Security Level: Public
    • None
    • windows 2008 R2 64 bit and centos 64-bit
    • Centos 64-bit

    Description

      Manifest info:
      <manifest>
      <remote name="couchbase" fetch="git://github.com/couchbase/"/>
      <remote name="membase" fetch="git://github.com/membase/"/>
      <remote name="apache" fetch="git://github.com/apache/"/>
      <remote name="erlang" fetch="git://github.com/erlang/"/>
      <default remote="couchbase" revision="master"/>
      <project name="tlm" path="tlm" revision="9f8a97b773c2b97cd63893a84a2fef2562c8860f">
      <copyfile src="Makefile.top" dest="Makefile"/>
      </project>
      <project name="bucket_engine" path="bucket_engine" revision="2a797a8d97f421587cce728f2e6aa2cd42c8fa26"/>
      <project name="ep-engine" path="ep-engine" revision="7fa9438353fa783d72e3e7f28e32703ab8922733"/>
      <project name="libconflate" path="libconflate" revision="c0d3e26a51f25a2b020713559cb344d43ce0b06c"/>
      <project name="libmemcached" path="libmemcached" revision="ca739a890349ac36dc79447e37da7caa9ae819f5" remote="membase"/>
      <project name="libvbucket" path="libvbucket" revision="408057ec55da3862ab8d75b1ed25d2848afd640f"/>
      <project name="couchbase-cli" path="couchbase-cli" revision="87dcaa935efb0eac4e75d529ab7e3c81b4439e61" remote="couchbase"/>
      <project name="memcached" path="memcached" revision="b6ceb46fc26ac6f1d6be7a5866d6c6c0f6e6d32a" remote="membase"/>
      <project name="moxi" path="moxi" revision="4b391021af7a453bf88716724d2c644916ebd969"/>
      <project name="ns_server" path="ns_server" revision="d77b8c4d9eb27fbd60778ca299cec29bca749e4c"/>
      <project name="portsigar" path="portsigar" revision="159b6179ea8a3c2075ee9eb2afa6f91c98c0fda6"/>
      <project name="sigar" path="sigar" revision="63a3cd1b316d2d4aa6dd31ce8fc66101b983e0b0"/>
      <project name="couchbase-examples" path="couchbase-examples" revision="cd9c8600589a1996c1ba6dbea9ac171b937d3379"/>
      <project name="couchbase-python-client" path="couchbase-python-client" revision="d443169c0694fca1be67d8f6934a8c50f0175ee7"/>
      <project name="couchdb" path="couchdb" revision="586e4bb73b92db4362192616370c4e3edb8c34a0"/>
      <project name="couchdbx-app" path="couchdbx-app" revision="833f5f2491c1d307d4d1ea291fd446a740df83f0"/>
      <project name="couchstore" path="couchstore" revision="abc2af1310ca375697e08aad4fa78e5e5d61adcf"/>
      <project name="geocouch" path="geocouch" revision="ed9ad43aa361df0829262fef811b5236331b44c8"/>
      <project name="testrunner" path="testrunner" revision="2ba2259a62e609b2bbaffc179be2d6c6be6ca61a"/>
      <project name="healthchecker" path="healthchecker" revision="72dab0d4f293e80644b38321f001b42846701890"/>
      <project name="otp" path="otp" revision="b6dc1a844eab061d0a7153d46e7e68296f15a504" remote="erlang"/>
      <project name="icu4c" path="icu4c" revision="26359393672c378f41f2103a8699c4357c894be7" remote="couchbase"/>
      <project name="snappy" path="snappy" revision="5681dde156e9d07adbeeab79666c9a9d7a10ec95" remote="couchbase"/>
      <project name="v8" path="v8" revision="447decb75060a106131ab4de934bcc374648e7f2" remote="couchbase"/>
      <project name="gperftools" path="gperftools" revision="44a584d1de8c89addfb4f1d0522bdbbbed83ba48" remote="couchbase"/>
      <project name="pysqlite" path="pysqlite" revision="0ff6e32ea05037fddef1eb41a648f2a2141009ea" remote="couchbase"/>
      </manifest>

      Rebalance 1 -> 4 hangs sometimes with status stuck at 0% and below logs repeating:

      {move_state,985,
      ['ns_1@10.3.3.100',undefined],
      ['ns_1@10.3.3.98','ns_1@10.3.3.96'],
      [

      {replica_building_stats,'ns_1@10.3.3.98',1,1,<<>>},
      {replica_building_stats,'ns_1@10.3.3.96',1,1,<<>>}]},
      {move_state,986,
      ['ns_1@10.3.3.100',undefined],
      ['ns_1@10.3.3.98','ns_1@10.3.3.96'],
      [{replica_building_stats,'ns_1@10.3.3.98',1,1,<<>>}

      ,

      {replica_building_stats,'ns_1@10.3.3.96',1,1,<<>>}]},
      {move_state,987,
      ['ns_1@10.3.3.100',undefined],
      ['ns_1@10.3.3.98','ns_1@10.3.3.96'],
      [{replica_building_stats,'ns_1@10.3.3.98',1,1,<<>>},
      {replica_building_stats,'ns_1@10.3.3.96',1,1,<<>>}

      ]},
      {move_state,988,
      ['ns_1@10.3.3.100',undefined],
      ['ns_1@10.3.3.98','ns_1@10.3.3.96'],
      [

      {replica_building_stats,'ns_1@10.3.3.98',1,1,<<>>}

      ,

      {replica_building_stats,'ns_1@10.3.3.96',1,1,<<>>}

      ]},

      Reproducer test:
      ./testrunner -i ../ini/vm-4nodes-sanity.ini -t warmupcluster.WarmUpClusterTest.test_warmUpCluster,num_of_docs=1000

      Attaching collect info

      Attachments

        For Gerrit Dashboard: MB-8231
        # Subject Branch Project Status CR V

        Activity

          People

            jin Jin Lim (Inactive)
            deepkaran.salooja Deepkaran Salooja
            Votes:
            0 Vote for this issue
            Watchers:
            7 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved:

              Gerrit Reviews

                There are no open Gerrit changes

                PagerDuty