time="2018-01-20T17:21:13Z" level=info msg="Obtaining resource lock" module=main
time="2018-01-20T17:21:13Z" level=info msg="Starting event recorder" module=main
time="2018-01-20T17:21:13Z" level=info msg="Attempting to be elected the couchbase-operator leader" module=main
time="2018-01-20T17:21:31Z" level=info msg="I'm the leader, attempt to start the operator" module=main
time="2018-01-20T17:21:31Z" level=info msg="Creating the couchbase-operator controller" module=main
time="2018-01-20T17:21:31Z" level=info msg="Event(v1.ObjectReference{Kind:\"Endpoints\", Namespace:\"default\", Name:\"couchbase-operator\", UID:\"128db932-f759-11e7-9929-0800275d0b06\", APIVersion:\"v1\", ResourceVersion:\"404310\", FieldPath:\"\"}): type: 'Normal' reason: 'LeaderElection' couchbase-operator-1917615544-lnps8 became leader" module=event_recorder
time="2018-01-20T17:21:31Z" level=info msg="CRD initialized, listening for events..." module=controller
time="2018-01-20T17:21:31Z" level=warning msg="Fail to handle add/update event: ignore failed cluster (test-couchbase-lnx92). Please delete its CR"
time="2018-01-20T17:21:43Z" level=info msg="Watching new cluster" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:21:44Z" level=warning msg="node init: failed with error Code: 0, Error: http://test-couchbase-bhg7m-0000.test-couchbase-bhg7m.default.svc:8091/node/controller/rename - Post http://test-couchbase-bhg7m-0000.test-couchbase-bhg7m.default.svc:8091/node/controller/rename: dial tcp 172.17.0.6:8091: getsockopt: connection refused ...retrying" cluster-name=test-couchbase-bhg7m module=retryutil
time="2018-01-20T17:21:49Z" level=warning msg="node init: failed with error Code: 0, Error: http://test-couchbase-bhg7m-0000.test-couchbase-bhg7m.default.svc:8091/node/controller/rename - Post http://test-couchbase-bhg7m-0000.test-couchbase-bhg7m.default.svc:8091/node/controller/rename: dial tcp 172.17.0.6:8091: getsockopt: connection refused ...retrying" cluster-name=test-couchbase-bhg7m module=retryutil
time="2018-01-20T17:21:57Z" level=info msg="start running..." cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:22:05Z" level=info msg="Start reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:22:05Z" level=info msg="server config test_config_1: test-couchbase-bhg7m-0000" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:22:05Z" level=info msg="running members: test-couchbase-bhg7m-0000" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:22:05Z" level=info msg="cluster membership: test-couchbase-bhg7m-0000" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:22:05Z" level=info msg="active nodes: test-couchbase-bhg7m-0000" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:22:06Z" level=info msg="added member (test-couchbase-bhg7m-0001)" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:22:06Z" level=warning msg="add node: failed with error Code: 400, Error: error - Failed to reach erlang port mapper. Failed to resolve address for \"test-couchbase-bhg7m-0001.test-couchbase-bhg7m.default.svc\".  The hostname may be incorrect or not resolvable. ...retrying" cluster-name=test-couchbase-bhg7m module=retryutil
time="2018-01-20T17:22:11Z" level=warning msg="add node: failed with error Code: 400, Error: error - Prepare join failed. Could not connect to \"test-couchbase-bhg7m-0001.test-couchbase-bhg7m.default.svc\" on port 8091.  This could be due to an incorrect host/port combination or a firewall in place between the servers. ...retrying" cluster-name=test-couchbase-bhg7m module=retryutil
time="2018-01-20T17:22:21Z" level=info msg="added member (test-couchbase-bhg7m-0002)" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:22:21Z" level=warning msg="add node: failed with error Code: 400, Error: error - Failed to reach erlang port mapper. Failed to resolve address for \"test-couchbase-bhg7m-0002.test-couchbase-bhg7m.default.svc\".  The hostname may be incorrect or not resolvable. ...retrying" cluster-name=test-couchbase-bhg7m module=retryutil
time="2018-01-20T17:22:26Z" level=warning msg="add node: failed with error Code: 400, Error: error - Prepare join failed. Could not connect to \"test-couchbase-bhg7m-0002.test-couchbase-bhg7m.default.svc\" on port 8091.  This could be due to an incorrect host/port combination or a firewall in place between the servers. ...retrying" cluster-name=test-couchbase-bhg7m module=retryutil
time="2018-01-20T17:22:35Z" level=info msg="added member (test-couchbase-bhg7m-0003)" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:22:35Z" level=warning msg="add node: failed with error Code: 400, Error: error - Failed to reach erlang port mapper. Could not connect to \"test-couchbase-bhg7m-0003.test-couchbase-bhg7m.default.svc\" on port \"4369\".  This could be due to an incorrect host/port combination or a firewall in place between the servers. ...retrying" cluster-name=test-couchbase-bhg7m module=retryutil
time="2018-01-20T17:22:40Z" level=warning msg="add node: failed with error Code: 400, Error: error - Prepare join failed. Could not connect to \"test-couchbase-bhg7m-0003.test-couchbase-bhg7m.default.svc\" on port 8091.  This could be due to an incorrect host/port combination or a firewall in place between the servers. ...retrying" cluster-name=test-couchbase-bhg7m module=retryutil
time="2018-01-20T17:22:50Z" level=info msg="added member (test-couchbase-bhg7m-0004)" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:22:51Z" level=warning msg="add node: failed with error Code: 400, Error: error - Prepare join failed. Could not connect to \"test-couchbase-bhg7m-0004.test-couchbase-bhg7m.default.svc\" on port 8091.  This could be due to an incorrect host/port combination or a firewall in place between the servers. ...retrying" cluster-name=test-couchbase-bhg7m module=retryutil
time="2018-01-20T17:22:55Z" level=warning msg="add node: failed with error Code: 400, Error: error - Prepare join failed. Could not connect to \"test-couchbase-bhg7m-0004.test-couchbase-bhg7m.default.svc\" on port 8091.  This could be due to an incorrect host/port combination or a firewall in place between the servers. ...retrying" cluster-name=test-couchbase-bhg7m module=retryutil
time="2018-01-20T17:23:06Z" level=info msg="Rebalance progress: 0.000000" cluster-name=test-couchbase-bhg7m module=retryutil
time="2018-01-20T17:23:10Z" level=info msg="Rebalance progress: 50.000000" cluster-name=test-couchbase-bhg7m module=retryutil
time="2018-01-20T17:23:18Z" level=info msg="Finish reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:23:26Z" level=info msg="Start reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:23:56Z" level=info msg="server config test_config_1: test-couchbase-bhg7m-0002,test-couchbase-bhg7m-0003,test-couchbase-bhg7m-0004" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:23:56Z" level=info msg="running members: test-couchbase-bhg7m-0002,test-couchbase-bhg7m-0003,test-couchbase-bhg7m-0004" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:23:56Z" level=info msg="cluster membership: test-couchbase-bhg7m-0000,test-couchbase-bhg7m-0001,test-couchbase-bhg7m-0002,test-couchbase-bhg7m-0003,test-couchbase-bhg7m-0004" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:23:56Z" level=info msg="active nodes: test-couchbase-bhg7m-0002,test-couchbase-bhg7m-0003,test-couchbase-bhg7m-0004" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:23:56Z" level=info msg="failed nodes: test-couchbase-bhg7m-0000,test-couchbase-bhg7m-0001" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:23:56Z" level=info msg="An auto-failover has taken place on http://test-couchbase-bhg7m-0000.test-couchbase-bhg7m.default.svc:8091, planning removal" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:23:56Z" level=info msg="An auto-failover has taken place on http://test-couchbase-bhg7m-0001.test-couchbase-bhg7m.default.svc:8091, planning removal" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:23:57Z" level=info msg="added member (test-couchbase-bhg7m-0005)" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:23:57Z" level=warning msg="add node: failed with error Code: 400, Error: error - Failed to reach erlang port mapper. Could not connect to \"test-couchbase-bhg7m-0005.test-couchbase-bhg7m.default.svc\" on port \"4369\".  This could be due to an incorrect host/port combination or a firewall in place between the servers. ...retrying" cluster-name=test-couchbase-bhg7m module=retryutil
time="2018-01-20T17:24:02Z" level=warning msg="add node: failed with error Code: 400, Error: error - Prepare join failed. Could not connect to \"test-couchbase-bhg7m-0005.test-couchbase-bhg7m.default.svc\" on port 8091.  This could be due to an incorrect host/port combination or a firewall in place between the servers. ...retrying" cluster-name=test-couchbase-bhg7m module=retryutil
time="2018-01-20T17:24:13Z" level=info msg="added member (test-couchbase-bhg7m-0006)" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:24:13Z" level=warning msg="add node: failed with error Code: 400, Error: error - Failed to reach erlang port mapper. Failed to resolve address for \"test-couchbase-bhg7m-0006.test-couchbase-bhg7m.default.svc\".  The hostname may be incorrect or not resolvable. ...retrying" cluster-name=test-couchbase-bhg7m module=retryutil
time="2018-01-20T17:24:18Z" level=warning msg="add node: failed with error Code: 400, Error: error - Prepare join failed. Could not connect to \"test-couchbase-bhg7m-0006.test-couchbase-bhg7m.default.svc\" on port 8091.  This could be due to an incorrect host/port combination or a firewall in place between the servers. ...retrying" cluster-name=test-couchbase-bhg7m module=retryutil
time="2018-01-20T17:24:28Z" level=info msg="Rebalance progress: 0.000000" cluster-name=test-couchbase-bhg7m module=retryutil
time="2018-01-20T17:24:38Z" level=info msg="removing one dead member" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:24:38Z" level=info msg="removing dead member \"test-couchbase-bhg7m-0001\"" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:24:38Z" level=warning msg="pod (test-couchbase-bhg7m-0001) not found while trying to delete it" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:24:38Z" level=info msg="deleted pod (test-couchbase-bhg7m-0001)" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:24:38Z" level=info msg="removed member (test-couchbase-bhg7m-0001)" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:24:38Z" level=info msg="Finish reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:24:46Z" level=info msg="Start reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:24:46Z" level=info msg="Finish reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:24:54Z" level=info msg="Start reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:24:54Z" level=info msg="Finish reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:25:02Z" level=info msg="Start reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:25:02Z" level=info msg="Finish reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:25:10Z" level=info msg="Start reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:25:10Z" level=info msg="Finish reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:25:18Z" level=info msg="Start reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:25:18Z" level=info msg="Finish reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:25:26Z" level=info msg="Start reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:25:26Z" level=info msg="Finish reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:25:34Z" level=info msg="Start reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:25:34Z" level=info msg="Finish reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:25:42Z" level=info msg="Start reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:25:42Z" level=info msg="Finish reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:25:50Z" level=info msg="Start reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:25:51Z" level=info msg="Finish reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:25:59Z" level=info msg="Start reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:25:59Z" level=info msg="Finish reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:26:07Z" level=info msg="Start reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:26:07Z" level=info msg="Finish reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:26:15Z" level=info msg="Start reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:26:15Z" level=info msg="Finish reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:26:23Z" level=info msg="Start reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:26:23Z" level=info msg="Finish reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:26:31Z" level=info msg="Start reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:26:31Z" level=info msg="Finish reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:26:39Z" level=info msg="Start reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:26:39Z" level=info msg="Finish reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:26:47Z" level=info msg="Start reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:26:47Z" level=info msg="Finish reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:26:55Z" level=info msg="Start reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:26:55Z" level=info msg="Finish reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:27:03Z" level=info msg="Start reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:27:03Z" level=info msg="Finish reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:27:11Z" level=info msg="Start reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:27:11Z" level=info msg="Finish reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:27:19Z" level=info msg="Start reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:27:19Z" level=info msg="Finish reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:27:27Z" level=info msg="Start reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:27:27Z" level=info msg="Finish reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:27:35Z" level=info msg="Start reconciling" cluster-name=test-couchbase-bhg7m module=cluster
time="2018-01-20T17:27:35Z" level=info msg="Finish reconciling" cluster-name=test-couchbase-bhg7m module=cluster