time="2018-04-04T10:36:16Z" level=info msg="Obtaining resource lock" module=main
time="2018-04-04T10:36:16Z" level=info msg="Starting event recorder" module=main
time="2018-04-04T10:36:16Z" level=info msg="Attempting to be elected the couchbase-operator leader" module=main
time="2018-04-04T10:36:33Z" level=info msg="I'm the leader, attempt to start the operator" module=main
time="2018-04-04T10:36:33Z" level=info msg="Creating the couchbase-operator controller" module=main
time="2018-04-04T10:36:33Z" level=info msg="Event(v1.ObjectReference{Kind:\"Endpoints\", Namespace:\"ashwin\", Name:\"couchbase-operator\", UID:\"5885aa53-2668-11e8-8299-6e85c3a27a72\", APIVersion:\"v1\", ResourceVersion:\"12169309\", FieldPath:\"\"}): type: 'Normal' reason: 'LeaderElection' couchbase-operator-6786d95858-ffrdd became leader" module=event_recorder
time="2018-04-04T10:36:33Z" level=info msg="CRD initialized, listening for events..." module=controller
time="2018-04-04T10:36:57Z" level=info msg="Watching new cluster" cluster-name=cb-example module=cluster
time="2018-04-04T10:38:33Z" level=error msg="cluster failed to setup: failed to create pod: " cluster-name=cb-example module=cluster
time="2018-04-04T10:38:33Z" level=error msg="failed to update cluster phase (Failed): failed to update CR status: Operation cannot be fulfilled on couchbaseclusters.couchbase.database.couchbase.com \"cb-example\": StorageError: invalid object, Code: 4, Key: /registry/couchbase.database.couchbase.com/couchbaseclusters/ashwin/cb-example, ResourceVersion: 0, AdditionalErrorMsg: Precondition failed: UID in precondition: 197128c6-37f4-11e8-8299-6e85c3a27a72, UID in object meta: " cluster-name=cb-example module=cluster
time="2018-04-04T10:38:47Z" level=info msg="Watching new cluster" cluster-name=cb-example module=cluster
time="2018-04-04T10:39:20Z" level=error msg="cluster failed to setup: failed to create pod: " cluster-name=cb-example module=cluster
time="2018-04-04T10:39:20Z" level=error msg="failed to update cluster phase (Failed): failed to update CR status: Operation cannot be fulfilled on couchbaseclusters.couchbase.database.couchbase.com \"cb-example\": StorageError: invalid object, Code: 4, Key: /registry/couchbase.database.couchbase.com/couchbaseclusters/ashwin/cb-example, ResourceVersion: 0, AdditionalErrorMsg: Precondition failed: UID in precondition: 5ae6fe91-37f4-11e8-8299-6e85c3a27a72, UID in object meta: " cluster-name=cb-example module=cluster
time="2018-04-04T10:40:26Z" level=info msg="Watching new cluster" cluster-name=cb-example module=cluster
time="2018-04-04T10:40:29Z" level=warning msg="node init: failed with error [Post http://cb-example-0000.cb-example.ashwin.svc:8091/node/controller/rename: dial tcp 10.36.0.3:8091: getsockopt: connection refused] ...retrying" cluster-name=cb-example module=retryutil
time="2018-04-04T10:40:34Z" level=warning msg="node init: failed with error [Post http://cb-example-0000.cb-example.ashwin.svc:8091/node/controller/rename: dial tcp 10.36.0.3:8091: getsockopt: connection refused] ...retrying" cluster-name=cb-example module=retryutil
time="2018-04-04T10:40:42Z" level=info msg="start running..." cluster-name=cb-example module=cluster
time="2018-04-04T10:40:50Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:40:50Z" level=info msg="server config all_services: cb-example-0000" cluster-name=cb-example module=cluster
time="2018-04-04T10:40:50Z" level=info msg="running members: cb-example-0000" cluster-name=cb-example module=cluster
time="2018-04-04T10:40:50Z" level=info msg="cluster membership: cb-example-0000" cluster-name=cb-example module=cluster
time="2018-04-04T10:40:50Z" level=info msg="active nodes: cb-example-0000" cluster-name=cb-example module=cluster
time="2018-04-04T10:40:51Z" level=info msg="added member (cb-example-0001)" cluster-name=cb-example module=cluster
time="2018-04-04T10:40:51Z" level=warning msg="add node: failed with error [Server Error 400 (cb-example-0000.cb-example.ashwin.svc:8091/controller/addNode): [error - Failed to reach erlang port mapper. Failed to resolve address for \"cb-example-0001.cb-example.ashwin.svc\".  The hostname may be incorrect or not resolvable.]] ...retrying" cluster-name=cb-example module=retryutil
time="2018-04-04T10:40:56Z" level=warning msg="add node: failed with error [Server Error 400 (cb-example-0000.cb-example.ashwin.svc:8091/controller/addNode): [error - Prepare join failed. Could not connect to \"cb-example-0001.cb-example.ashwin.svc\" on port 8091.  This could be due to an incorrect host/port combination or a firewall in place between the servers.]] ...retrying" cluster-name=cb-example module=retryutil
time="2018-04-04T10:41:06Z" level=info msg="added member (cb-example-0002)" cluster-name=cb-example module=cluster
time="2018-04-04T10:41:06Z" level=warning msg="add node: failed with error [Server Error 400 (cb-example-0001.cb-example.ashwin.svc:8091/controller/addNode): [error - Prepare join failed. Could not connect to \"cb-example-0002.cb-example.ashwin.svc\" on port 8091.  This could be due to an incorrect host/port combination or a firewall in place between the servers.]], [Server Error 400 (cb-example-0000.cb-example.ashwin.svc:8091/controller/addNode): [error - Prepare join failed. Could not connect to \"cb-example-0002.cb-example.ashwin.svc\" on port 8091.  This could be due to an incorrect host/port combination or a firewall in place between the servers.]] ...retrying" cluster-name=cb-example module=retryutil
time="2018-04-04T10:41:11Z" level=warning msg="add node: failed with error [Server Error 400 (cb-example-0001.cb-example.ashwin.svc:8091/controller/addNode): [error - Prepare join failed. Could not connect to \"cb-example-0002.cb-example.ashwin.svc\" on port 8091.  This could be due to an incorrect host/port combination or a firewall in place between the servers.]], [Server Error 400 (cb-example-0000.cb-example.ashwin.svc:8091/controller/addNode): [error - Prepare join failed. Could not connect to \"cb-example-0002.cb-example.ashwin.svc\" on port 8091.  This could be due to an incorrect host/port combination or a firewall in place between the servers.]] ...retrying" cluster-name=cb-example module=retryutil
time="2018-04-04T10:41:19Z" level=info msg="Rebalance progress: 0.000000" cluster-name=cb-example module=retryutil
time="2018-04-04T10:41:27Z" level=warning msg="create bucket: failed with error bucket default is not ready: creation pending ...retrying" cluster-name=cb-example module=retryutil
time="2018-04-04T10:41:32Z" level=info msg="Created bucket default" cluster-name=cb-example module=cluster
time="2018-04-04T10:41:32Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:41:40Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:41:40Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:41:48Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:41:48Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:41:56Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:41:56Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:42:04Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:42:04Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:42:12Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:42:12Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:42:20Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:42:20Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:42:28Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:42:28Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:42:36Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:42:36Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:42:44Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:42:44Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:42:52Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:42:52Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:43:00Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:43:00Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:43:08Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:43:08Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:43:16Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:43:16Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:43:24Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:43:24Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:43:32Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:43:32Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:43:40Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:43:40Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:43:48Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:43:48Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:43:56Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:43:56Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:44:04Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:44:04Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:44:12Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:44:13Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:44:21Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:44:21Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:44:29Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:44:29Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:44:37Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:44:37Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:44:45Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:44:45Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:44:52Z" level=info msg="Watching new cluster" cluster-name=cb-example-1 module=cluster
time="2018-04-04T10:44:53Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:44:53Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:44:54Z" level=warning msg="node init: failed with error [Post http://cb-example-1-0000.cb-example-1.ashwin.svc:8091/node/controller/rename: dial tcp 10.39.0.4:8091: getsockopt: connection refused] ...retrying" cluster-name=cb-example-1 module=retryutil
time="2018-04-04T10:44:59Z" level=warning msg="node init: failed with error [Post http://cb-example-1-0000.cb-example-1.ashwin.svc:8091/node/controller/rename: dial tcp 10.39.0.4:8091: getsockopt: connection refused] ...retrying" cluster-name=cb-example-1 module=retryutil
time="2018-04-04T10:45:01Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:45:01Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:45:07Z" level=info msg="start running..." cluster-name=cb-example-1 module=cluster
time="2018-04-04T10:45:09Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:45:09Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:45:15Z" level=info msg="Start reconciling" cluster-name=cb-example-1 module=cluster
time="2018-04-04T10:45:15Z" level=info msg="server config all_services: cb-example-1-0000" cluster-name=cb-example-1 module=cluster
time="2018-04-04T10:45:15Z" level=info msg="running members: cb-example-1-0000" cluster-name=cb-example-1 module=cluster
time="2018-04-04T10:45:15Z" level=info msg="cluster membership: cb-example-1-0000" cluster-name=cb-example-1 module=cluster
time="2018-04-04T10:45:15Z" level=info msg="active nodes: cb-example-1-0000" cluster-name=cb-example-1 module=cluster
time="2018-04-04T10:45:16Z" level=info msg="added member (cb-example-1-0001)" cluster-name=cb-example-1 module=cluster
time="2018-04-04T10:45:16Z" level=warning msg="add node: failed with error [Server Error 400 (cb-example-1-0000.cb-example-1.ashwin.svc:8091/controller/addNode): [error - Failed to reach erlang port mapper. Failed to resolve address for \"cb-example-1-0001.cb-example-1.ashwin.svc\".  The hostname may be incorrect or not resolvable.]] ...retrying" cluster-name=cb-example-1 module=retryutil
time="2018-04-04T10:45:17Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:45:17Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:45:21Z" level=warning msg="add node: failed with error [Server Error 400 (cb-example-1-0000.cb-example-1.ashwin.svc:8091/controller/addNode): [error - Prepare join failed. Could not connect to \"cb-example-1-0001.cb-example-1.ashwin.svc\" on port 8091.  This could be due to an incorrect host/port combination or a firewall in place between the servers.]] ...retrying" cluster-name=cb-example-1 module=retryutil
time="2018-04-04T10:45:25Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:45:25Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:45:30Z" level=info msg="added member (cb-example-1-0002)" cluster-name=cb-example-1 module=cluster
time="2018-04-04T10:45:30Z" level=warning msg="add node: failed with error [Server Error 400 (cb-example-1-0001.cb-example-1.ashwin.svc:8091/controller/addNode): [error - Prepare join failed. Could not connect to \"cb-example-1-0002.cb-example-1.ashwin.svc\" on port 8091.  This could be due to an incorrect host/port combination or a firewall in place between the servers.]], [Server Error 400 (cb-example-1-0000.cb-example-1.ashwin.svc:8091/controller/addNode): [error - Prepare join failed. Could not connect to \"cb-example-1-0002.cb-example-1.ashwin.svc\" on port 8091.  This could be due to an incorrect host/port combination or a firewall in place between the servers.]] ...retrying" cluster-name=cb-example-1 module=retryutil
time="2018-04-04T10:45:33Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:45:33Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:45:35Z" level=warning msg="add node: failed with error [Server Error 400 (cb-example-1-0001.cb-example-1.ashwin.svc:8091/controller/addNode): [error - Prepare join failed. Could not connect to \"cb-example-1-0002.cb-example-1.ashwin.svc\" on port 8091.  This could be due to an incorrect host/port combination or a firewall in place between the servers.]], [Server Error 400 (cb-example-1-0000.cb-example-1.ashwin.svc:8091/controller/addNode): [error - Prepare join failed. Could not connect to \"cb-example-1-0002.cb-example-1.ashwin.svc\" on port 8091.  This could be due to an incorrect host/port combination or a firewall in place between the servers.]] ...retrying" cluster-name=cb-example-1 module=retryutil
time="2018-04-04T10:45:41Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:45:41Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:45:44Z" level=info msg="Rebalance progress: 0.000000" cluster-name=cb-example-1 module=retryutil
time="2018-04-04T10:45:49Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:45:49Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:45:52Z" level=warning msg="create bucket: failed with error bucket default is not ready: creation pending ...retrying" cluster-name=cb-example-1 module=retryutil
time="2018-04-04T10:45:57Z" level=warning msg="create bucket: failed with error bucket default is not ready: warmup ...retrying" cluster-name=cb-example-1 module=retryutil
time="2018-04-04T10:45:57Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:45:57Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:46:02Z" level=info msg="Created bucket default" cluster-name=cb-example-1 module=cluster
time="2018-04-04T10:46:02Z" level=info msg="Finish reconciling" cluster-name=cb-example-1 module=cluster
time="2018-04-04T10:46:05Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:46:05Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:46:10Z" level=info msg="Start reconciling" cluster-name=cb-example-1 module=cluster
time="2018-04-04T10:46:10Z" level=info msg="Finish reconciling" cluster-name=cb-example-1 module=cluster
time="2018-04-04T10:46:13Z" level=info msg="Start reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:46:13Z" level=info msg="Finish reconciling" cluster-name=cb-example module=cluster
time="2018-04-04T10:46:18Z" level=info msg="Start reconciling" cluster-name=cb-example-1 module=cluster
time="2018-04-04T10:46:18Z" level=info msg="Finish reconciling" cluster-name=cb-example-1 module=cluster