Uploaded image for project: 'Couchbase Server'
  1. Couchbase Server
  2. MB-51567

Rebalance fails with CBAS after upgrading to latest 7.1 Re

    XMLWordPrintable

Details

    • Bug
    • Resolution: User Error
    • Critical
    • 7.1.0
    • 7.1.0
    • analytics
    • None
    • Untriaged
    • 1
    • Unknown

    Description

      Env
      2 Node Cluster running all services on-prem on Couchbase VMs.
      OS: Ubuntu 20.04
      No Firewalls

      Action
      Upgraded to Enterprise Edition 7.1.0 build 2506 
      Nodes had to be Rebalanced (button enabled).
      Performed Rebalance

      Result
      Gets stuck in trying to Rebalance Analytics for 5 minutes and Fails

      Report
      {
      "data": {
      "stageInfo": {
      "backup":

      { "startTime": false, "completedTime": false, "timeTaken": false }

      ,
      "analytics": {
      "startTime": "2022-03-24T00:13:28.331Z",
      "completedTime": false,
      "timeTaken": 300078
      },
      "eventing": {
      "startTime": false,
      "completedTime": false,
      "timeTaken": false
      },
      "search": {
      "totalProgress": 100,
      "perNodeProgress":

      { "ns_1@172.23.120.94": 1, "ns_1@172.23.120.57": 1 }

      ,
      "startTime": "2022-03-24T00:13:26.798Z",
      "completedTime": "2022-03-24T00:13:26.942Z",
      "timeTaken": 144
      },
      "index": {
      "totalProgress": 100,
      "perNodeProgress":

      { "ns_1@172.23.120.94": 1, "ns_1@172.23.120.57": 1 }

      ,
      "startTime": "2022-03-24T00:13:26.942Z",
      "completedTime": "2022-03-24T00:13:28.331Z",
      "timeTaken": 1389
      },
      "data": {
      "totalProgress": 100,
      "perNodeProgress":

      { "ns_1@172.23.120.94": 1, "ns_1@172.23.120.57": 1 }

      ,
      "startTime": "2022-03-24T00:13:24.418Z",
      "completedTime": "2022-03-24T00:13:26.798Z",
      "timeTaken": 2380
      },
      "query": {
      "startTime": false,
      "completedTime": false,
      "timeTaken": false
      }
      },
      "rebalanceId": "bb6f9e13b6444fb91f29170996980b0b",
      "nodesInfo": {
      "active_nodes": [
      "ns_1@172.23.120.57",
      "ns_1@172.23.120.94"
      ],
      "keep_nodes": [
      "ns_1@172.23.120.57",
      "ns_1@172.23.120.94"
      ],
      "eject_nodes": [],
      "delta_nodes": [],
      "failed_nodes": []
      },
      "masterNode": "ns_1@172.23.120.94",
      "startTime": "2022-03-24T00:13:24.413Z",
      "completedTime": "2022-03-24T00:18:28.409Z",
      "timeTaken": 303996,
      "completionMessage": "Rebalance exited with reason {service_rebalance_failed,cbas,\n {worker_died,\n {'EXIT',<0.18736.4>,\n {rebalance_failed,\n

      {service_error,\n <<\"wait for condition on http://172.23.120.57:9111/analytics/cluster timed out\">>}

      }}}}."
      },
      "status": 200,
      "config": {
      "method": "GET",
      "transformRequest": [
      null
      ],
      "transformResponse": [
      null
      ],
      "jsonpCallbackParam": "callback",
      "url": "/logs/rebalanceReport",
      "cache": true,
      "headers":

      { "Accept": "application/json, text/plain, */*", "invalid-auth-response": "on", "Cache-Control": "no-cache", "Pragma": "no-cache", "ns-server-ui": "yes" }

      ,
      "timeout": {
      "$$state": {
      "status": 0,
      "pending": [
      [
      {
      "$$state":

      { "status": 0 }

      },
      null,
      null,
      null
      ]
      ]
      }
      }
      },
      "statusText": "OK",
      "xhrStatus": "complete"
      }

      Attachments

        Activity

          People

            krishna.doddi Krishna Doddi
            krishna.doddi Krishna Doddi
            Votes:
            0 Vote for this issue
            Watchers:
            5 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved:

              PagerDuty