{"level":"info","ts":"2024-05-31T11:34:48Z","logger":"main","msg":"couchbase-operator","version":"2.6.4 (build 121)","revision":"b34476b11042031e170bf9d3c83b82045b3fe125"}
{"level":"info","ts":"2024-05-31T11:34:48Z","logger":"controller-runtime.metrics","msg":"Starting metrics server"}
{"level":"info","ts":"2024-05-31T11:34:48Z","msg":"attempting to acquire leader lease default/couchbase-operator..."}
{"level":"info","ts":"2024-05-31T11:34:48Z","logger":"controller-runtime.metrics","msg":"Serving metrics server","bindAddress":"0.0.0.0:8383","secure":false}
{"level":"info","ts":"2024-05-31T11:35:04Z","msg":"successfully acquired lease default/couchbase-operator"}
{"level":"info","ts":"2024-05-31T11:35:04Z","msg":"Starting EventSource","controller":"couchbase-controller","source":"kind source: *v2.CouchbaseCluster"}
{"level":"info","ts":"2024-05-31T11:35:04Z","msg":"Starting Controller","controller":"couchbase-controller"}
{"level":"info","ts":"2024-05-31T11:35:05Z","msg":"Starting workers","controller":"couchbase-controller","worker count":4}
{"level":"info","ts":"2024-05-31T11:35:05Z","logger":"cluster","msg":"Watching new cluster","cluster":"default/cb-example"}
{"level":"info","ts":"2024-05-31T11:35:28Z","logger":"cluster","msg":"Couchbase client starting","cluster":"default/cb-example"}
{"level":"info","ts":"2024-05-31T11:35:28Z","logger":"cluster","msg":"Janitor starting","cluster":"default/cb-example"}
{"level":"info","ts":"2024-05-31T11:35:43Z","logger":"cluster","msg":"Running","cluster":"default/cb-example"}
{"level":"info","ts":"2024-05-31T11:37:55Z","logger":"cluster","msg":"Resource updated","cluster":"default/cb-example","diff":"-{v2.ClusterStatus}.Members.Ready[7->?]:cb-example-0007;+{v2.ClusterStatus}.Members.Unready:[cb-example-0007]"}
{"level":"info","ts":"2024-05-31T11:37:55Z","logger":"cluster","msg":"Cluster status","cluster":"default/cb-example","balance":"balanced","rebalancing":false}
{"level":"info","ts":"2024-05-31T11:37:55Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0000","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:37:55Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0001","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:37:55Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0002","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:37:55Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0003","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:37:55Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0004","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:37:55Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0005","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:37:55Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0006","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:37:55Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0007","version":"7.2.5","class":"index-query","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:37:55Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0008","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:37:55Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0009","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:37:55Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0000","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:37:55Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0001","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:37:55Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0002","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:37:55Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0004","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:37:55Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0003","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:37:55Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0005","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:37:55Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0009","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:37:55Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0006","class":"index-query","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:37:55Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0008","class":"index-query","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:37:55Z","logger":"cluster","msg":"Resource updated","cluster":"default/cb-example","diff":"-{v2.ClusterStatus}.Conditions[0->?]:{Type:Available Status:True LastUpdateTime:2024-05-31T10:26:22Z LastTransitionTime:2024-05-31T10:26:22Z Reason:Available Message:};+{v2.ClusterStatus}.Conditions[?->0]:{Type:Available Status:False LastUpdateTime:2024-05-31T11:37:55Z LastTransitionTime:2024-05-31T11:37:55Z Reason:PartiallyAvailable Message:The following nodes are down and not serving requests: cb-example-0007}"}
{"level":"info","ts":"2024-05-31T11:37:55Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0007","recovery_in":34.99999906}
{"level":"info","ts":"2024-05-31T11:37:55Z","logger":"cluster","msg":"Reconciliation failed","cluster":"default/cb-example","error":"reconcile was blocked from running: waiting for pod failover","stack":"github.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).handleDownNodes\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:435\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).exec\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:319\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcileMembers\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:268\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:177\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).runReconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:492\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).Update\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:535\ngithub.com/couchbase/couchbase-operator/pkg/controller.(*CouchbaseClusterReconciler).Reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/controller/controller.go:90\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:227"}
{"level":"info","ts":"2024-05-31T11:37:55Z","logger":"cluster","msg":"Resource updated","cluster":"default/cb-example","diff":"+{v2.ClusterStatus}.Conditions[?->2]:{Type:Error Status:True LastUpdateTime:2024-05-31T11:37:55Z LastTransitionTime:2024-05-31T11:37:55Z Reason:ErrorEncountered Message:reconcile was blocked from running: waiting for pod failover}"}
{"level":"info","ts":"2024-05-31T11:37:56Z","logger":"cluster","msg":"Cluster status","cluster":"default/cb-example","balance":"balanced","rebalancing":false}
{"level":"info","ts":"2024-05-31T11:37:56Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0000","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:37:56Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0001","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:37:56Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0002","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:37:56Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0003","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:37:56Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0004","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:37:56Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0005","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:37:56Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0006","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:37:56Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0007","version":"7.2.5","class":"index-query","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:37:56Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0008","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:37:56Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0009","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:37:56Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0000","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:37:56Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0001","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:37:56Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0002","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:37:56Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0004","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:37:56Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0003","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:37:56Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0005","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:37:56Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0009","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:37:56Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0006","class":"index-query","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:37:56Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0008","class":"index-query","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:37:56Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0007","recovery_in":34.308316862}
{"level":"info","ts":"2024-05-31T11:37:56Z","logger":"cluster","msg":"Reconciliation failed","cluster":"default/cb-example","error":"reconcile was blocked from running: waiting for pod failover","stack":"github.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).handleDownNodes\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:435\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).exec\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:319\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcileMembers\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:268\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:177\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).runReconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:492\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).Update\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:535\ngithub.com/couchbase/couchbase-operator/pkg/controller.(*CouchbaseClusterReconciler).Reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/controller/controller.go:90\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:227"}
{"level":"info","ts":"2024-05-31T11:38:06Z","logger":"cluster","msg":"Cluster status","cluster":"default/cb-example","balance":"unbalanced","rebalancing":false}
{"level":"info","ts":"2024-05-31T11:38:06Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0000","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:38:06Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0001","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:38:06Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0002","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:38:06Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0003","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:38:06Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0004","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:38:06Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0005","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:38:06Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0006","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:38:06Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0007","version":"7.2.5","class":"index-query","managed":true,"status":"Failed"}
{"level":"info","ts":"2024-05-31T11:38:06Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0008","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:38:06Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0009","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:38:06Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0000","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:38:06Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0001","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:38:06Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0002","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:38:06Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0004","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:38:06Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0003","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:38:06Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0005","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:38:06Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0009","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:38:06Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0006","class":"index-query","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:38:06Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0008","class":"index-query","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:38:06Z","logger":"cluster","msg":"Pods failed over","cluster":"default/cb-example"}
{"level":"info","ts":"2024-05-31T11:38:06Z","logger":"kubernetes","msg":"Creating pod","cluster":"default/cb-example","name":"cb-example-0007","image":"couchbase/server:7.2.5"}
{"level":"info","ts":"2024-05-31T11:44:49Z","logger":"cluster","msg":"Reconciliation failed","cluster":"default/cb-example","error":"reconcile was blocked from running: recovering node cb-example-0007","stack":"github.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).handleFailedNodes\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:661\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).exec\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:319\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcileMembers\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:268\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:177\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).runReconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:492\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).Update\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:535\ngithub.com/couchbase/couchbase-operator/pkg/controller.(*CouchbaseClusterReconciler).Reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/controller/controller.go:90\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:227"}
{"level":"info","ts":"2024-05-31T11:44:49Z","logger":"cluster","msg":"Resource updated","cluster":"default/cb-example","diff":"{v2.ClusterStatus}.Conditions[2].LastUpdateTime:2024-05-31T11:37:55Z->2024-05-31T11:44:49Z;{v2.ClusterStatus}.Conditions[2].Message:reconcile was blocked from running: waiting for pod failover->reconcile was blocked from running: recovering node cb-example-0007"}
{"level":"info","ts":"2024-05-31T11:44:49Z","logger":"cluster","msg":"Resource updated","cluster":"default/cb-example","diff":"-{v2.ClusterStatus}.Members.Ready[0->?]:cb-example-0000;-{v2.ClusterStatus}.Members.Ready[1->?]:cb-example-0001;-{v2.ClusterStatus}.Members.Ready[2->?]:cb-example-0002;-{v2.ClusterStatus}.Members.Ready[3->?]:cb-example-0003;-{v2.ClusterStatus}.Members.Ready[4->?]:cb-example-0004;-{v2.ClusterStatus}.Members.Ready[5->?]:cb-example-0005;+{v2.ClusterStatus}.Members.Unready[?->0]:cb-example-0000;+{v2.ClusterStatus}.Members.Unready[?->1]:cb-example-0001;+{v2.ClusterStatus}.Members.Unready[?->2]:cb-example-0002;+{v2.ClusterStatus}.Members.Unready[?->3]:cb-example-0003;+{v2.ClusterStatus}.Members.Unready[?->4]:cb-example-0004;+{v2.ClusterStatus}.Members.Unready[?->5]:cb-example-0005"}
{"level":"info","ts":"2024-05-31T11:44:49Z","logger":"cluster","msg":"Cluster status","cluster":"default/cb-example","balance":"unbalanced","rebalancing":false}
{"level":"info","ts":"2024-05-31T11:44:49Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0000","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:44:49Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0001","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:44:49Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0002","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:44:49Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0003","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:44:49Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0004","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:44:49Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0005","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:44:49Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0006","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:44:49Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0007","version":"7.2.5","class":"index-query","managed":true,"status":"AddBack"}
{"level":"info","ts":"2024-05-31T11:44:49Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0008","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:44:49Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0009","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:44:49Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0000","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:44:49Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0001","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:44:49Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0002","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:44:49Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0004","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:44:49Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0003","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:44:49Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0005","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:44:49Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0007","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:44:49Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0009","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:44:49Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0006","class":"index-query","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:44:49Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0008","class":"index-query","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:44:49Z","logger":"cluster","msg":"Resource updated","cluster":"default/cb-example","diff":"{v2.ClusterStatus}.Conditions[0].LastUpdateTime:2024-05-31T11:37:55Z->2024-05-31T11:44:49Z;{v2.ClusterStatus}.Conditions[0].Message:The following nodes are down and not serving requests: cb-example-0007->The following nodes are down and not serving requests: cb-example-0000, cb-example-0001, cb-example-0002, cb-example-0003, cb-example-0004, cb-example-0005"}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0000","recovery_in":34.999999328}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0004","recovery_in":34.999999704}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0005","recovery_in":34.999999705}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0001","recovery_in":34.999999606}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0002","recovery_in":34.99999969}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0003","recovery_in":34.99999977}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"cluster","msg":"Reconciliation failed","cluster":"default/cb-example","error":"reconcile was blocked from running: waiting for pod failover","stack":"github.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).handleDownNodes\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:435\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).exec\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:319\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcileMembers\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:268\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:177\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).runReconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:492\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).Update\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:535\ngithub.com/couchbase/couchbase-operator/pkg/controller.(*CouchbaseClusterReconciler).Reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/controller/controller.go:90\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:227"}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"cluster","msg":"Resource updated","cluster":"default/cb-example","diff":"{v2.ClusterStatus}.Conditions[2].LastUpdateTime:2024-05-31T11:44:49Z->2024-05-31T11:44:55Z;{v2.ClusterStatus}.Conditions[2].Message:reconcile was blocked from running: recovering node cb-example-0007->reconcile was blocked from running: waiting for pod failover"}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"cluster","msg":"Cluster status","cluster":"default/cb-example","balance":"unbalanced","rebalancing":false}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0000","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0001","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0002","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0003","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0004","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0005","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0006","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0007","version":"7.2.5","class":"index-query","managed":true,"status":"AddBack"}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0008","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0009","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0000","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0001","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0002","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0004","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0003","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0005","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0007","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0009","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0006","class":"index-query","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0008","class":"index-query","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0002","recovery_in":34.756471131}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0004","recovery_in":34.756409248}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0000","recovery_in":34.756371169}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0001","recovery_in":34.756408024}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0003","recovery_in":34.756412136}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0005","recovery_in":34.756397973}
{"level":"info","ts":"2024-05-31T11:44:55Z","logger":"cluster","msg":"Reconciliation failed","cluster":"default/cb-example","error":"reconcile was blocked from running: waiting for pod failover","stack":"github.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).handleDownNodes\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:435\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).exec\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:319\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcileMembers\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:268\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:177\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).runReconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:492\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).Update\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:535\ngithub.com/couchbase/couchbase-operator/pkg/controller.(*CouchbaseClusterReconciler).Reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/controller/controller.go:90\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:227"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"cluster","msg":"Resource updated","cluster":"default/cb-example","diff":"+{v2.ClusterStatus}.Members.Ready[?->1]:cb-example-0007;-{v2.ClusterStatus}.Members.Unready[6->?]:cb-example-0007"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"cluster","msg":"Cluster status","cluster":"default/cb-example","balance":"unbalanced","rebalancing":false}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0000","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0001","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0002","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0003","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0004","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0005","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0006","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0007","version":"7.2.5","class":"index-query","managed":true,"status":"AddBack"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0008","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0009","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0000","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0001","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0002","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0004","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0003","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0005","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0007","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0009","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0006","class":"index-query","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0008","class":"index-query","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0005","recovery_in":30.303479253}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0000","recovery_in":30.303408553}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0002","recovery_in":30.30344972}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0003","recovery_in":30.303449574}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0001","recovery_in":30.303439014}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0004","recovery_in":30.303427714}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"cluster","msg":"Reconciliation failed","cluster":"default/cb-example","error":"reconcile was blocked from running: waiting for pod failover","stack":"github.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).handleDownNodes\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:435\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).exec\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:319\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcileMembers\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:268\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:177\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).runReconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:492\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).Update\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:535\ngithub.com/couchbase/couchbase-operator/pkg/controller.(*CouchbaseClusterReconciler).Reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/controller/controller.go:90\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:227"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"cluster","msg":"Cluster status","cluster":"default/cb-example","balance":"unbalanced","rebalancing":false}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0000","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0001","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0002","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0003","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0004","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0005","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0006","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0007","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0008","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0009","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0000","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0001","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0002","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0004","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0003","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0005","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0007","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0009","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0006","class":"index-query","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:44:59Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0008","class":"index-query","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:45:00Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0000","recovery_in":29.916838333}
{"level":"info","ts":"2024-05-31T11:45:00Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0001","recovery_in":29.916844031}
{"level":"info","ts":"2024-05-31T11:45:00Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0002","recovery_in":29.916841792}
{"level":"info","ts":"2024-05-31T11:45:00Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0004","recovery_in":29.916826032}
{"level":"info","ts":"2024-05-31T11:45:00Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0005","recovery_in":29.916827416}
{"level":"info","ts":"2024-05-31T11:45:00Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0003","recovery_in":29.916835296}
{"level":"info","ts":"2024-05-31T11:45:00Z","logger":"cluster","msg":"Reconciliation failed","cluster":"default/cb-example","error":"reconcile was blocked from running: waiting for pod failover","stack":"github.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).handleDownNodes\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:435\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).exec\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:319\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcileMembers\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:268\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:177\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).runReconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:492\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).Update\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:535\ngithub.com/couchbase/couchbase-operator/pkg/controller.(*CouchbaseClusterReconciler).Reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/controller/controller.go:90\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:227"}
{"level":"info","ts":"2024-05-31T11:45:09Z","logger":"cluster","msg":"Resource updated","cluster":"default/cb-example","diff":"-{v2.ClusterStatus}.Members.Ready[1->?]:cb-example-0007;+{v2.ClusterStatus}.Members.Unready[?->6]:cb-example-0007"}
{"level":"info","ts":"2024-05-31T11:45:09Z","logger":"cluster","msg":"Cluster status","cluster":"default/cb-example","balance":"unbalanced","rebalancing":false}
{"level":"info","ts":"2024-05-31T11:45:09Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0000","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:09Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0001","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:09Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0002","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:09Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0003","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:09Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0004","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:09Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0005","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:09Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0006","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:45:09Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0007","version":"7.2.5","class":"index-query","managed":true,"status":"AddBack"}
{"level":"info","ts":"2024-05-31T11:45:09Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0008","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:45:09Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0009","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:45:09Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0000","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:45:09Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0001","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:45:09Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0002","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:45:09Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0004","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:45:09Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0003","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:45:09Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0005","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:45:09Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0007","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:45:09Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0009","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:45:09Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0006","class":"index-query","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:45:09Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0008","class":"index-query","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:45:09Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0001","recovery_in":20.083959234}
{"level":"info","ts":"2024-05-31T11:45:09Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0003","recovery_in":20.083939728}
{"level":"info","ts":"2024-05-31T11:45:09Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0000","recovery_in":20.083886676}
{"level":"info","ts":"2024-05-31T11:45:09Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0002","recovery_in":20.083928909}
{"level":"info","ts":"2024-05-31T11:45:09Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0004","recovery_in":20.083913344}
{"level":"info","ts":"2024-05-31T11:45:09Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0005","recovery_in":20.083914665}
{"level":"info","ts":"2024-05-31T11:45:09Z","logger":"cluster","msg":"Reconciliation failed","cluster":"default/cb-example","error":"reconcile was blocked from running: waiting for pod failover","stack":"github.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).handleDownNodes\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:435\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).exec\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:319\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcileMembers\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:268\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:177\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).runReconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:492\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).Update\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:535\ngithub.com/couchbase/couchbase-operator/pkg/controller.(*CouchbaseClusterReconciler).Reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/controller/controller.go:90\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:227"}
{"level":"info","ts":"2024-05-31T11:45:09Z","logger":"cluster","msg":"Resource updated","cluster":"default/cb-example","diff":"+{v2.ClusterStatus}.Members.Ready[?->1]:cb-example-0007;-{v2.ClusterStatus}.Members.Unready[6->?]:cb-example-0007"}
{"level":"info","ts":"2024-05-31T11:45:10Z","logger":"cluster","msg":"Cluster status","cluster":"default/cb-example","balance":"unbalanced","rebalancing":false}
{"level":"info","ts":"2024-05-31T11:45:10Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0000","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:10Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0001","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:10Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0002","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:10Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0003","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:10Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0004","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:10Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0005","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:10Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0006","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:45:10Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0007","version":"7.2.5","class":"index-query","managed":true,"status":"AddBack"}
{"level":"info","ts":"2024-05-31T11:45:10Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0008","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:45:10Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0009","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:45:10Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0000","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:45:10Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0001","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:45:10Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0002","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:45:10Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0004","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:45:10Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0003","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:45:10Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0005","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:45:10Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0007","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:45:10Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0009","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:45:10Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0006","class":"index-query","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:45:10Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0008","class":"index-query","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:45:10Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0001","recovery_in":19.395961944}
{"level":"info","ts":"2024-05-31T11:45:10Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0002","recovery_in":19.395934378}
{"level":"info","ts":"2024-05-31T11:45:10Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0003","recovery_in":19.39593353}
{"level":"info","ts":"2024-05-31T11:45:10Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0005","recovery_in":19.395918918}
{"level":"info","ts":"2024-05-31T11:45:10Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0000","recovery_in":19.395877437}
{"level":"info","ts":"2024-05-31T11:45:10Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0004","recovery_in":19.395907617}
{"level":"info","ts":"2024-05-31T11:45:10Z","logger":"cluster","msg":"Reconciliation failed","cluster":"default/cb-example","error":"reconcile was blocked from running: waiting for pod failover","stack":"github.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).handleDownNodes\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:435\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).exec\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:319\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcileMembers\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:268\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:177\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).runReconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:492\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).Update\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:535\ngithub.com/couchbase/couchbase-operator/pkg/controller.(*CouchbaseClusterReconciler).Reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/controller/controller.go:90\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:227"}
{"level":"info","ts":"2024-05-31T11:45:11Z","logger":"cluster","msg":"Cluster status","cluster":"default/cb-example","balance":"unbalanced","rebalancing":false}
{"level":"info","ts":"2024-05-31T11:45:11Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0000","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:11Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0001","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:11Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0002","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:11Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0003","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:11Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0004","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:11Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0005","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:11Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0006","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:45:11Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0007","version":"7.2.5","class":"index-query","managed":true,"status":"AddBack"}
{"level":"info","ts":"2024-05-31T11:45:11Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0008","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:45:11Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0009","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:45:11Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0000","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:45:11Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0001","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:45:11Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0002","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:45:11Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0004","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:45:11Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0003","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:45:11Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0005","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:45:11Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0007","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:45:11Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0009","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:45:11Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0006","class":"index-query","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:45:11Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0008","class":"index-query","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:45:12Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0004","recovery_in":17.987705564}
{"level":"info","ts":"2024-05-31T11:45:12Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0005","recovery_in":17.987676448}
{"level":"info","ts":"2024-05-31T11:45:12Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0002","recovery_in":17.987679777}
{"level":"info","ts":"2024-05-31T11:45:12Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0003","recovery_in":17.987679317}
{"level":"info","ts":"2024-05-31T11:45:12Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0000","recovery_in":17.987627135}
{"level":"info","ts":"2024-05-31T11:45:12Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0001","recovery_in":17.987665618}
{"level":"info","ts":"2024-05-31T11:45:12Z","logger":"cluster","msg":"Reconciliation failed","cluster":"default/cb-example","error":"reconcile was blocked from running: waiting for pod failover","stack":"github.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).handleDownNodes\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:435\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).exec\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:319\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcileMembers\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:268\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:177\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).runReconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:492\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).Update\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:535\ngithub.com/couchbase/couchbase-operator/pkg/controller.(*CouchbaseClusterReconciler).Reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/controller/controller.go:90\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:227"}
{"level":"info","ts":"2024-05-31T11:45:19Z","logger":"cluster","msg":"Resource updated","cluster":"default/cb-example","diff":"-{v2.ClusterStatus}.Members.Ready[1->?]:cb-example-0007;+{v2.ClusterStatus}.Members.Unready[?->6]:cb-example-0007"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Cluster status","cluster":"default/cb-example","balance":"unbalanced","rebalancing":false}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0000","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0001","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0002","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0003","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0004","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0005","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0006","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0007","version":"7.2.5","class":"index-query","managed":true,"status":"AddBack"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0008","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0009","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0000","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0001","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0002","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0004","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0003","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0005","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0007","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0009","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0006","class":"index-query","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0008","class":"index-query","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0005","recovery_in":9.86829415}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0000","recovery_in":9.868230764}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0002","recovery_in":9.86827199}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0004","recovery_in":9.868256419}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0001","recovery_in":9.868261617}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0003","recovery_in":9.868265934}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Reconciliation failed","cluster":"default/cb-example","error":"reconcile was blocked from running: waiting for pod failover","stack":"github.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).handleDownNodes\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:435\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).exec\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:319\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcileMembers\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:268\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:177\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).runReconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:492\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).Update\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:535\ngithub.com/couchbase/couchbase-operator/pkg/controller.(*CouchbaseClusterReconciler).Reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/controller/controller.go:90\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:227"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Cluster status","cluster":"default/cb-example","balance":"unbalanced","rebalancing":false}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0000","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0001","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0002","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0003","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0004","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0005","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0006","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0007","version":"7.2.5","class":"index-query","managed":true,"status":"AddBack"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0008","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0009","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0000","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0001","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0002","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0004","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0003","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0005","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0007","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0009","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0006","class":"index-query","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0008","class":"index-query","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0000","recovery_in":9.388144399}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0001","recovery_in":9.38815494}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0002","recovery_in":9.388154259}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0003","recovery_in":9.388154108}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0004","recovery_in":9.388134718}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0005","recovery_in":9.388135368}
{"level":"info","ts":"2024-05-31T11:45:20Z","logger":"cluster","msg":"Reconciliation failed","cluster":"default/cb-example","error":"reconcile was blocked from running: waiting for pod failover","stack":"github.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).handleDownNodes\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:435\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).exec\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:319\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcileMembers\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:268\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:177\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).runReconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:492\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).Update\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:535\ngithub.com/couchbase/couchbase-operator/pkg/controller.(*CouchbaseClusterReconciler).Reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/controller/controller.go:90\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:227"}
{"level":"info","ts":"2024-05-31T11:45:30Z","logger":"cluster","msg":"Cluster status","cluster":"default/cb-example","balance":"unbalanced","rebalancing":false}
{"level":"info","ts":"2024-05-31T11:45:30Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0000","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:30Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0001","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:30Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0002","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:30Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0003","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:30Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0004","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:30Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0005","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:45:30Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0006","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:45:30Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0007","version":"7.2.5","class":"index-query","managed":true,"status":"AddBack"}
{"level":"info","ts":"2024-05-31T11:45:30Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0008","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:45:30Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0009","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:45:30Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0000","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:45:30Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0001","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:45:30Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0002","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:45:30Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0004","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:45:30Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0003","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:45:30Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0005","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:45:30Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0007","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:45:30Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0009","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:45:30Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0006","class":"index-query","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:45:30Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0008","class":"index-query","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:45:30Z","logger":"cluster","msg":"Node failed","cluster":"default/cb-example","name":"cb-example-0003"}
{"level":"info","ts":"2024-05-31T11:45:31Z","logger":"kubernetes","msg":"Creating pod","cluster":"default/cb-example","name":"cb-example-0003","image":"couchbase/server:7.2.5"}
{"level":"info","ts":"2024-05-31T11:55:31Z","logger":"cluster","msg":"Member creation failed","cluster":"default/cb-example","name":"cb-example-0003"}
{"level":"info","ts":"2024-05-31T11:55:31Z","logger":"cluster","msg":"Pod deleted","cluster":"default/cb-example","name":"cb-example-0003"}
{"level":"info","ts":"2024-05-31T11:55:31Z","logger":"cluster","msg":"Reconciliation failed","cluster":"default/cb-example","error":"pod recovery failed for member cb-example-0003: timeout: unexpected kubernetes error: pod phase Pending not Running as expected","stack":"github.com/couchbase/couchbase-operator/pkg/util/k8sutil.WaitForPod.func1\n\tgithub.com/couchbase/couchbase-operator/pkg/util/k8sutil/k8sutil.go:281\ngithub.com/couchbase/couchbase-operator/pkg/util/retryutil.Retry\n\tgithub.com/couchbase/couchbase-operator/pkg/util/retryutil/retryutil.go:14\ngithub.com/couchbase/couchbase-operator/pkg/util/k8sutil.WaitForPod\n\tgithub.com/couchbase/couchbase-operator/pkg/util/k8sutil/k8sutil.go:287\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).waitForCreatePod\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/pod.go:112\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).createPod\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/pod.go:43\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).recreatePod\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/pod.go:81\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).handleDownNodes\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:457\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).exec\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:319\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcileMembers\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:268\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:177\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).runReconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:492\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).Update\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:535\ngithub.com/couchbase/couchbase-operator/pkg/controller.(*CouchbaseClusterReconciler).Reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/controller/controller.go:90\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:227"}
{"level":"info","ts":"2024-05-31T11:55:31Z","logger":"cluster","msg":"Resource updated","cluster":"default/cb-example","diff":"{v2.ClusterStatus}.Conditions[2].LastUpdateTime:2024-05-31T11:44:55Z->2024-05-31T11:55:31Z;{v2.ClusterStatus}.Conditions[2].Message:reconcile was blocked from running: waiting for pod failover->pod recovery failed for member cb-example-0003: timeout: unexpected kubernetes error: pod phase Pending not Running as expected"}
{"level":"info","ts":"2024-05-31T11:55:31Z","logger":"cluster","msg":"Cluster status","cluster":"default/cb-example","balance":"unbalanced","rebalancing":false}
{"level":"info","ts":"2024-05-31T11:55:31Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0000","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:55:31Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0001","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:55:31Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0002","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:55:31Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0003","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:55:31Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0004","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:55:31Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0005","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T11:55:31Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0006","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:55:31Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0007","version":"7.2.5","class":"index-query","managed":true,"status":"AddBack"}
{"level":"info","ts":"2024-05-31T11:55:31Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0008","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:55:31Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0009","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T11:55:31Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0000","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:55:31Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0001","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:55:31Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0002","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:55:31Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0004","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:55:31Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0005","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:55:31Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0007","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:55:31Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0009","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T11:55:31Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0006","class":"index-query","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T11:55:31Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0008","class":"index-query","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T11:55:37Z","logger":"cluster","msg":"Node failed","cluster":"default/cb-example","name":"cb-example-0000"}
{"level":"info","ts":"2024-05-31T11:55:37Z","logger":"kubernetes","msg":"Creating pod","cluster":"default/cb-example","name":"cb-example-0000","image":"couchbase/server:7.2.5"}
{"level":"info","ts":"2024-05-31T12:02:18Z","logger":"cluster","msg":"Pod recovering","cluster":"default/cb-example","name":"cb-example-0000"}
{"level":"info","ts":"2024-05-31T12:02:18Z","logger":"cluster","msg":"Node failed","cluster":"default/cb-example","name":"cb-example-0003"}
{"level":"info","ts":"2024-05-31T12:02:18Z","logger":"kubernetes","msg":"Creating pod","cluster":"default/cb-example","name":"cb-example-0003","image":"couchbase/server:7.2.5"}
{"level":"info","ts":"2024-05-31T12:03:20Z","logger":"cluster","msg":"Pod recovering","cluster":"default/cb-example","name":"cb-example-0003"}
{"level":"info","ts":"2024-05-31T12:03:20Z","logger":"cluster","msg":"Node failed","cluster":"default/cb-example","name":"cb-example-0004"}
{"level":"info","ts":"2024-05-31T12:03:21Z","logger":"kubernetes","msg":"Creating pod","cluster":"default/cb-example","name":"cb-example-0004","image":"couchbase/server:7.2.5"}
{"level":"info","ts":"2024-05-31T12:04:06Z","logger":"cluster","msg":"Pod recovering","cluster":"default/cb-example","name":"cb-example-0004"}
{"level":"info","ts":"2024-05-31T12:04:06Z","logger":"cluster","msg":"Node failed","cluster":"default/cb-example","name":"cb-example-0001"}
{"level":"info","ts":"2024-05-31T12:04:06Z","logger":"kubernetes","msg":"Creating pod","cluster":"default/cb-example","name":"cb-example-0001","image":"couchbase/server:7.2.5"}
{"level":"info","ts":"2024-05-31T12:04:48Z","logger":"cluster","msg":"Pod recovering","cluster":"default/cb-example","name":"cb-example-0001"}
{"level":"info","ts":"2024-05-31T12:04:48Z","logger":"cluster","msg":"Node failed","cluster":"default/cb-example","name":"cb-example-0002"}
{"level":"info","ts":"2024-05-31T12:04:49Z","logger":"kubernetes","msg":"Creating pod","cluster":"default/cb-example","name":"cb-example-0002","image":"couchbase/server:7.2.5"}
{"level":"info","ts":"2024-05-31T12:14:49Z","logger":"cluster","msg":"Member creation failed","cluster":"default/cb-example","name":"cb-example-0002"}
{"level":"info","ts":"2024-05-31T12:14:49Z","logger":"cluster","msg":"Pod deleted","cluster":"default/cb-example","name":"cb-example-0002"}
{"level":"info","ts":"2024-05-31T12:14:49Z","logger":"cluster","msg":"Reconciliation failed","cluster":"default/cb-example","error":"pod recovery failed for member cb-example-0002: timeout: unexpected kubernetes error: pod phase Pending not Running as expected","stack":"github.com/couchbase/couchbase-operator/pkg/util/k8sutil.WaitForPod.func1\n\tgithub.com/couchbase/couchbase-operator/pkg/util/k8sutil/k8sutil.go:281\ngithub.com/couchbase/couchbase-operator/pkg/util/retryutil.Retry\n\tgithub.com/couchbase/couchbase-operator/pkg/util/retryutil/retryutil.go:14\ngithub.com/couchbase/couchbase-operator/pkg/util/k8sutil.WaitForPod\n\tgithub.com/couchbase/couchbase-operator/pkg/util/k8sutil/k8sutil.go:287\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).waitForCreatePod\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/pod.go:112\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).createPod\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/pod.go:43\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).recreatePod\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/pod.go:81\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).handleDownNodes\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:457\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).exec\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:319\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcileMembers\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:268\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:177\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).runReconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:492\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).Update\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:535\ngithub.com/couchbase/couchbase-operator/pkg/controller.(*CouchbaseClusterReconciler).Reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/controller/controller.go:90\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:227"}
{"level":"info","ts":"2024-05-31T12:14:49Z","logger":"cluster","msg":"Resource updated","cluster":"default/cb-example","diff":"{v2.ClusterStatus}.Conditions[2].LastUpdateTime:2024-05-31T11:55:31Z->2024-05-31T12:14:49Z;{v2.ClusterStatus}.Conditions[2].Message:pod recovery failed for member cb-example-0003: timeout: unexpected kubernetes error: pod phase Pending not Running as expected->pod recovery failed for member cb-example-0002: timeout: unexpected kubernetes error: pod phase Pending not Running as expected"}
{"level":"info","ts":"2024-05-31T12:14:49Z","logger":"cluster","msg":"Deleted terminated pod","cluster":"default/cb-example","name":"cb-example-0005"}
{"level":"info","ts":"2024-05-31T12:14:49Z","logger":"cluster","msg":"Cluster status","cluster":"default/cb-example","balance":"unbalanced","rebalancing":false}
{"level":"info","ts":"2024-05-31T12:14:49Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0000","version":"7.2.5","class":"data-only","managed":true,"status":"Warmup"}
{"level":"info","ts":"2024-05-31T12:14:49Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0001","version":"7.2.5","class":"data-only","managed":true,"status":"Warmup"}
{"level":"info","ts":"2024-05-31T12:14:49Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0002","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T12:14:49Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0003","version":"7.2.5","class":"data-only","managed":true,"status":"Warmup"}
{"level":"info","ts":"2024-05-31T12:14:49Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0004","version":"7.2.5","class":"data-only","managed":true,"status":"Warmup"}
{"level":"info","ts":"2024-05-31T12:14:49Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0005","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T12:14:49Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0006","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:14:49Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0007","version":"7.2.5","class":"index-query","managed":true,"status":"AddBack"}
{"level":"info","ts":"2024-05-31T12:14:49Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0008","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:14:49Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0009","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:14:49Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0000","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:14:49Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0001","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:14:49Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0004","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T12:14:49Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0003","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T12:14:49Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0007","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:14:49Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0009","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:14:49Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0006","class":"index-query","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T12:14:49Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0008","class":"index-query","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T12:14:49Z","logger":"cluster","msg":"Resource updated","cluster":"default/cb-example","diff":"{v2.ClusterStatus}.Conditions[0].LastUpdateTime:2024-05-31T11:44:49Z->2024-05-31T12:14:49Z;{v2.ClusterStatus}.Conditions[0].Message:The following nodes are down and not serving requests: cb-example-0000, cb-example-0001, cb-example-0002, cb-example-0003, cb-example-0004, cb-example-0005->The following nodes are down and not serving requests: cb-example-0002, cb-example-0005"}
{"level":"info","ts":"2024-05-31T12:14:51Z","logger":"cluster","msg":"Node failed","cluster":"default/cb-example","name":"cb-example-0002"}
{"level":"info","ts":"2024-05-31T12:14:51Z","logger":"kubernetes","msg":"Creating pod","cluster":"default/cb-example","name":"cb-example-0002","image":"couchbase/server:7.2.5"}
{"level":"info","ts":"2024-05-31T12:24:02Z","logger":"cluster","msg":"Pod recovering","cluster":"default/cb-example","name":"cb-example-0002"}
{"level":"info","ts":"2024-05-31T12:24:02Z","logger":"cluster","msg":"Node failed","cluster":"default/cb-example","name":"cb-example-0005"}
{"level":"info","ts":"2024-05-31T12:24:02Z","logger":"kubernetes","msg":"Creating pod","cluster":"default/cb-example","name":"cb-example-0005","image":"couchbase/server:7.2.5"}
{"level":"info","ts":"2024-05-31T12:24:24Z","logger":"cluster","msg":"Pod recovering","cluster":"default/cb-example","name":"cb-example-0005"}
{"level":"info","ts":"2024-05-31T12:24:24Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"waiting for cluster recovery"}
{"level":"info","ts":"2024-05-31T12:24:24Z","logger":"cluster","msg":"Resource updated","cluster":"default/cb-example","diff":"-{v2.ClusterStatus}.Conditions[2->?]:{Type:Error Status:True LastUpdateTime:2024-05-31T12:14:49Z LastTransitionTime:2024-05-31T11:37:55Z Reason:ErrorEncountered Message:pod recovery failed for member cb-example-0002: timeout: unexpected kubernetes error: pod phase Pending not Running as expected}"}
{"level":"info","ts":"2024-05-31T12:24:24Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:24:34Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:24:44Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:24:55Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:25:05Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:25:15Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:25:25Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:25:35Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:25:45Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:25:55Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:26:05Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:26:15Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:26:25Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:26:35Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:26:45Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:26:56Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:27:06Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:27:16Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:27:26Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:27:36Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:27:46Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:27:56Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:28:06Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:28:16Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:28:26Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:28:36Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:28:46Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:28:57Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:29:07Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:29:17Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:29:27Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:29:37Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:29:47Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:29:57Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:30:07Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:30:17Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:30:27Z","logger":"cluster","msg":"Resource updated","cluster":"default/cb-example","diff":"+{v2.ClusterStatus}.Members.Ready[?->0]:cb-example-0000;+{v2.ClusterStatus}.Members.Ready[?->1]:cb-example-0001;+{v2.ClusterStatus}.Members.Ready[?->2]:cb-example-0002;+{v2.ClusterStatus}.Members.Ready[?->3]:cb-example-0003;+{v2.ClusterStatus}.Members.Ready[?->4]:cb-example-0004;+{v2.ClusterStatus}.Members.Ready[?->5]:cb-example-0005;-{v2.ClusterStatus}.Members.Unready[0->?]:cb-example-0000;-{v2.ClusterStatus}.Members.Unready[1->?]:cb-example-0001;-{v2.ClusterStatus}.Members.Unready[2->?]:cb-example-0002;-{v2.ClusterStatus}.Members.Unready[3->?]:cb-example-0003;-{v2.ClusterStatus}.Members.Unready[4->?]:cb-example-0004;-{v2.ClusterStatus}.Members.Unready[5->?]:cb-example-0005"}
{"level":"info","ts":"2024-05-31T12:30:27Z","logger":"cluster","msg":"Cluster status","cluster":"default/cb-example","balance":"unbalanced","rebalancing":false}
{"level":"info","ts":"2024-05-31T12:30:27Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0000","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:30:27Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0001","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:30:27Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0002","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:30:27Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0003","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:30:27Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0004","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:30:27Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0005","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:30:27Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0006","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:30:27Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0007","version":"7.2.5","class":"index-query","managed":true,"status":"AddBack"}
{"level":"info","ts":"2024-05-31T12:30:27Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0008","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:30:27Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0009","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:30:27Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0000","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:30:27Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0001","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:30:27Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0002","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T12:30:27Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0004","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T12:30:27Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0003","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T12:30:27Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0005","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T12:30:27Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0007","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:30:27Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0009","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:30:27Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0006","class":"index-query","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T12:30:27Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0008","class":"index-query","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T12:30:27Z","logger":"cluster","msg":"Setting recovery type","cluster":"default/cb-example","name":"cb-example-0007","type":"full"}
{"level":"info","ts":"2024-05-31T12:30:29Z","logger":"cluster","msg":"Resource updated","cluster":"default/cb-example","diff":"-{v2.ClusterStatus}.Conditions[1->?]:{Type:Balanced Status:True LastUpdateTime:2024-05-31T10:26:11Z LastTransitionTime:2024-05-31T10:26:11Z Reason:Balanced Message:Data is equally distributed across all nodes in the cluster};+{v2.ClusterStatus}.Conditions[?->1]:{Type:Balanced Status:False LastUpdateTime:2024-05-31T12:30:29Z LastTransitionTime:2024-05-31T12:30:29Z Reason:Unbalanced Message:The operator is attempting to rebalance the data to correct this issue}"}
{"level":"info","ts":"2024-05-31T12:30:30Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:30:34Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":36}
{"level":"info","ts":"2024-05-31T12:30:38Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":60}
{"level":"info","ts":"2024-05-31T12:30:42Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":61.99999999999999}
{"level":"info","ts":"2024-05-31T12:30:46Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":61.99999999999999}
{"level":"info","ts":"2024-05-31T12:30:50Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":61.99999999999999}
{"level":"info","ts":"2024-05-31T12:30:54Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":61.99999999999999}
{"level":"info","ts":"2024-05-31T12:30:58Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":61.99999999999999}
{"level":"info","ts":"2024-05-31T12:31:02Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":61.99999999999999}
{"level":"info","ts":"2024-05-31T12:31:06Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":61.99999999999999}
{"level":"info","ts":"2024-05-31T12:31:10Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":61.99999999999999}
{"level":"info","ts":"2024-05-31T12:31:14Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":61.99999999999999}
{"level":"info","ts":"2024-05-31T12:31:18Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":61.99999999999999}
{"level":"info","ts":"2024-05-31T12:31:22Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":61.99999999999999}
{"level":"info","ts":"2024-05-31T12:31:26Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":61.99999999999999}
{"level":"info","ts":"2024-05-31T12:31:30Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":61.99999999999999}
{"level":"info","ts":"2024-05-31T12:31:34Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":61.99999999999999}
{"level":"info","ts":"2024-05-31T12:31:38Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":61.99999999999999}
{"level":"info","ts":"2024-05-31T12:31:42Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":61.99999999999999}
{"level":"info","ts":"2024-05-31T12:31:46Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":61.99999999999999}
{"level":"info","ts":"2024-05-31T12:31:50Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":61.99999999999999}
{"level":"info","ts":"2024-05-31T12:31:54Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":61.99999999999999}
{"level":"info","ts":"2024-05-31T12:31:58Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":80}
{"level":"info","ts":"2024-05-31T12:32:02Z","logger":"cluster","msg":"Rebalance completed successfully","cluster":"default/cb-example"}
{"level":"info","ts":"2024-05-31T12:32:02Z","logger":"cluster","msg":"Resource updated","cluster":"default/cb-example","diff":"-{v2.ClusterStatus}.Conditions[1->?]:{Type:Balanced Status:False LastUpdateTime:2024-05-31T12:30:29Z LastTransitionTime:2024-05-31T12:30:29Z Reason:Unbalanced Message:The operator is attempting to rebalance the data to correct this issue};+{v2.ClusterStatus}.Conditions[?->1]:{Type:Balanced Status:True LastUpdateTime:2024-05-31T12:32:02Z LastTransitionTime:2024-05-31T12:32:02Z Reason:Balanced Message:Data is equally distributed across all nodes in the cluster}"}
{"level":"info","ts":"2024-05-31T12:32:02Z","logger":"cluster","msg":"Resource updated","cluster":"default/cb-example","diff":"+{v2.ClusterStatus}.Members.Ready[?->7]:cb-example-0007;-{v2.ClusterStatus}.Members.Unready:[cb-example-0007]"}
{"level":"info","ts":"2024-05-31T12:32:03Z","logger":"cluster","msg":"Resource updated","cluster":"default/cb-example","diff":"-{v2.ClusterStatus}.Conditions[0->?]:{Type:Available Status:False LastUpdateTime:2024-05-31T12:14:49Z LastTransitionTime:2024-05-31T11:37:55Z Reason:PartiallyAvailable Message:The following nodes are down and not serving requests: cb-example-0002, cb-example-0005};+{v2.ClusterStatus}.Conditions[?->0]:{Type:Available Status:True LastUpdateTime:2024-05-31T12:32:03Z LastTransitionTime:2024-05-31T12:32:03Z Reason:Available Message:};+{v2.ClusterStatus}.Autoscalers:[]"}
{"level":"info","ts":"2024-05-31T12:33:06Z","logger":"cluster","msg":"Resource updated","cluster":"default/cb-example","diff":"-{v2.ClusterStatus}.Members.Ready[1->?]:cb-example-0001;+{v2.ClusterStatus}.Members.Unready:[cb-example-0001]"}
{"level":"info","ts":"2024-05-31T12:33:06Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:33:07Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:33:17Z","logger":"cluster","msg":"Cluster status","cluster":"default/cb-example","balance":"unbalanced","rebalancing":false}
{"level":"info","ts":"2024-05-31T12:33:17Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0000","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:33:17Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0001","version":"7.2.5","class":"data-only","managed":true,"status":"Failed"}
{"level":"info","ts":"2024-05-31T12:33:17Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0002","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:33:17Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0003","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:33:17Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0004","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:33:17Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0005","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:33:17Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0006","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:33:17Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0007","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:33:17Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0008","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:33:17Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0009","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:33:17Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0000","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:33:17Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0002","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T12:33:17Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0004","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T12:33:17Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0003","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T12:33:17Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0005","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T12:33:17Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0007","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:33:17Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0009","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:33:17Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0006","class":"index-query","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T12:33:17Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0008","class":"index-query","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T12:33:17Z","logger":"cluster","msg":"Pods failed over","cluster":"default/cb-example"}
{"level":"info","ts":"2024-05-31T12:33:17Z","logger":"kubernetes","msg":"Creating pod","cluster":"default/cb-example","name":"cb-example-0001","image":"couchbase/server:7.2.5"}
{"level":"info","ts":"2024-05-31T12:39:58Z","logger":"cluster","msg":"Reconciliation failed","cluster":"default/cb-example","error":"reconcile was blocked from running: recovering node cb-example-0001","stack":"github.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).handleFailedNodes\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:661\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).exec\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:319\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcileMembers\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:268\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:177\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).runReconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:492\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).Update\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:535\ngithub.com/couchbase/couchbase-operator/pkg/controller.(*CouchbaseClusterReconciler).Reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/controller/controller.go:90\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:227"}
{"level":"info","ts":"2024-05-31T12:39:58Z","logger":"cluster","msg":"Resource updated","cluster":"default/cb-example","diff":"+{v2.ClusterStatus}.Conditions[?->2]:{Type:Error Status:True LastUpdateTime:2024-05-31T12:39:58Z LastTransitionTime:2024-05-31T12:39:58Z Reason:ErrorEncountered Message:reconcile was blocked from running: recovering node cb-example-0001}"}
{"level":"info","ts":"2024-05-31T12:39:58Z","logger":"cluster","msg":"Cluster status","cluster":"default/cb-example","balance":"unbalanced","rebalancing":false}
{"level":"info","ts":"2024-05-31T12:39:58Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0000","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:39:58Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0001","version":"7.2.5","class":"data-only","managed":true,"status":"AddBack"}
{"level":"info","ts":"2024-05-31T12:39:58Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0002","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:39:58Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0003","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:39:58Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0004","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:39:58Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0005","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:39:58Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0006","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:39:58Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0007","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:39:58Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0008","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:39:58Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0009","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:39:58Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0000","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:39:58Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0001","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:39:58Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0002","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T12:39:58Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0004","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T12:39:58Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0003","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T12:39:58Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0005","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T12:39:58Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0007","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:39:58Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0009","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:39:58Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0006","class":"index-query","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T12:39:58Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0008","class":"index-query","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T12:39:58Z","logger":"cluster","msg":"Setting recovery type","cluster":"default/cb-example","name":"cb-example-0001","type":"delta"}
{"level":"info","ts":"2024-05-31T12:40:00Z","logger":"cluster","msg":"Resource updated","cluster":"default/cb-example","diff":"-{v2.ClusterStatus}.Conditions[1->?]:{Type:Balanced Status:True LastUpdateTime:2024-05-31T12:32:02Z LastTransitionTime:2024-05-31T12:32:02Z Reason:Balanced Message:Data is equally distributed across all nodes in the cluster};+{v2.ClusterStatus}.Conditions[?->1]:{Type:Balanced Status:False LastUpdateTime:2024-05-31T12:40:00Z LastTransitionTime:2024-05-31T12:40:00Z Reason:Unbalanced Message:The operator is attempting to rebalance the data to correct this issue}"}
{"level":"info","ts":"2024-05-31T12:40:01Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:40:05Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:40:09Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:40:13Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:40:17Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:40:21Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:40:25Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:40:29Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:40:33Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:40:37Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:40:41Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:40:45Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:40:49Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:40:53Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:40:57Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:41:01Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:41:05Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:41:09Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:41:13Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:41:17Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:41:21Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:41:25Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:41:29Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:41:33Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:41:37Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:41:41Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:41:45Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:41:49Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:41:53Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:41:57Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:42:01Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:42:05Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:42:09Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:42:13Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:42:17Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:42:21Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:42:25Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:42:29Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:42:33Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:42:37Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:42:41Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:42:45Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:42:49Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:42:53Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:42:57Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:43:01Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:43:05Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:43:09Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:43:13Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:43:17Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:43:21Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:43:25Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:43:29Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:43:33Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:43:37Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:43:41Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:43:45Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:43:49Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:43:53Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:43:57Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:44:01Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:44:05Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:44:09Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:44:13Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:44:17Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:44:21Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:44:25Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:44:29Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:44:33Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:44:37Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:44:41Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:44:45Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:44:49Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:44:53Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:44:57Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:45:01Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:45:05Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:45:09Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:45:13Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:45:17Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:45:21Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:45:25Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:45:29Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:45:33Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:45:37Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:45:41Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:45:45Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:45:50Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:45:54Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:45:58Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:46:02Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:46:06Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:46:10Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:46:14Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0}
{"level":"info","ts":"2024-05-31T12:46:18Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0.158843072112566}
{"level":"info","ts":"2024-05-31T12:46:22Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0.2909354439450199}
{"level":"info","ts":"2024-05-31T12:46:26Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0.4492248062015504}
{"level":"info","ts":"2024-05-31T12:46:30Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0.6091752980261871}
{"level":"info","ts":"2024-05-31T12:46:34Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":0.8477167611230539}
{"level":"info","ts":"2024-05-31T12:46:38Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":1.16484919549215}
{"level":"info","ts":"2024-05-31T12:46:42Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":1.429033939157058}
{"level":"info","ts":"2024-05-31T12:46:46Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":1.666467982541854}
{"level":"info","ts":"2024-05-31T12:46:50Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":2.38264608168849}
{"level":"info","ts":"2024-05-31T12:46:54Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":3.016910950426682}
{"level":"info","ts":"2024-05-31T12:46:58Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":3.598781838316722}
{"level":"info","ts":"2024-05-31T12:47:02Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":4.207403426486874}
{"level":"info","ts":"2024-05-31T12:47:06Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":5.605895381408376}
{"level":"info","ts":"2024-05-31T12:47:10Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":6.63422252621979}
{"level":"info","ts":"2024-05-31T12:47:14Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":7.163145723405641}
{"level":"info","ts":"2024-05-31T12:47:18Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":8.223206957201484}
{"level":"info","ts":"2024-05-31T12:47:22Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":9.073138557748681}
{"level":"info","ts":"2024-05-31T12:47:26Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":9.683421275486939}
{"level":"info","ts":"2024-05-31T12:47:30Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":13.19760601915185}
{"level":"info","ts":"2024-05-31T12:47:34Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":16.68614748224871}
{"level":"info","ts":"2024-05-31T12:47:38Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":17.02892319718585}
{"level":"info","ts":"2024-05-31T12:47:42Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":17.53331053351573}
{"level":"info","ts":"2024-05-31T12:47:46Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":17.82479968731679}
{"level":"info","ts":"2024-05-31T12:47:50Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":18.16923653182203}
{"level":"info","ts":"2024-05-31T12:47:54Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":18.51422708618331}
{"level":"info","ts":"2024-05-31T12:47:58Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":19.01916813236923}
{"level":"info","ts":"2024-05-31T12:48:02Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":19.76043580222787}
{"level":"info","ts":"2024-05-31T12:48:06Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":20.12996221744512}
{"level":"info","ts":"2024-05-31T12:48:10Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":20.8155136473194}
{"level":"info","ts":"2024-05-31T12:48:14Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":27.5}
{"level":"info","ts":"2024-05-31T12:48:18Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":27.73854146309687}
{"level":"info","ts":"2024-05-31T12:48:22Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":28.00217249690574}
{"level":"info","ts":"2024-05-31T12:48:26Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":28.1610155690183}
{"level":"info","ts":"2024-05-31T12:48:30Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":28.31985864113087}
{"level":"info","ts":"2024-05-31T12:48:34Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":28.53109569409159}
{"level":"info","ts":"2024-05-31T12:48:38Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":28.77019086704449}
{"level":"info","ts":"2024-05-31T12:48:42Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":29.03548303042147}
{"level":"info","ts":"2024-05-31T12:48:46Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":29.32641847436649}
{"level":"info","ts":"2024-05-31T12:48:50Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":29.67085531887173}
{"level":"info","ts":"2024-05-31T12:48:54Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":29.96179076281676}
{"level":"info","ts":"2024-05-31T12:48:58Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":30.33242459774608}
{"level":"info","ts":"2024-05-31T12:49:02Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":30.67686144225132}
{"level":"info","ts":"2024-05-31T12:49:06Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":30.83570451436389}
{"level":"info","ts":"2024-05-31T12:49:10Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":31.15449807830108}
{"level":"info","ts":"2024-05-31T12:49:14Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":31.34009185069376}
{"level":"info","ts":"2024-05-31T12:49:18Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":31.62991987492671}
{"level":"info","ts":"2024-05-31T12:49:22Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":31.73526154647905}
{"level":"info","ts":"2024-05-31T12:49:26Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":31.97380300957592}
{"level":"info","ts":"2024-05-31T12:49:30Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":33.0796983909843}
{"level":"info","ts":"2024-05-31T12:49:34Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":33.3438831346492}
{"level":"info","ts":"2024-05-31T12:49:38Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":33.58131717803401}
{"level":"info","ts":"2024-05-31T12:49:42Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":33.79310794085076}
{"level":"info","ts":"2024-05-31T12:49:46Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":33.9781480033874}
{"level":"info","ts":"2024-05-31T12:49:50Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":34.13643736564393}
{"level":"info","ts":"2024-05-31T12:49:54Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":34.24233274705232}
{"level":"info","ts":"2024-05-31T12:49:58Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":34.48032050029314}
{"level":"info","ts":"2024-05-31T12:50:02Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":34.61185916226956}
{"level":"info","ts":"2024-05-31T12:50:06Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":34.84984691551039}
{"level":"info","ts":"2024-05-31T12:50:10Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":35.08838837860726}
{"level":"info","ts":"2024-05-31T12:50:14Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":35.30017914142401}
{"level":"info","ts":"2024-05-31T12:50:18Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":35.40607452283238}
{"level":"info","ts":"2024-05-31T12:50:22Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":35.56491759494496}
{"level":"info","ts":"2024-05-31T12:50:26Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":35.72376066705752}
{"level":"info","ts":"2024-05-31T12:50:30Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":36.01524982085858}
{"level":"info","ts":"2024-05-31T12:50:34Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":36.3061852648036}
{"level":"info","ts":"2024-05-31T12:50:38Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":36.43827763663605}
{"level":"info","ts":"2024-05-31T12:50:42Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":36.7843756107094}
{"level":"info","ts":"2024-05-31T12:50:46Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":37.1561168653508}
{"level":"info","ts":"2024-05-31T12:50:50Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":38.5}
{"level":"info","ts":"2024-05-31T12:50:54Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":39.02892319718585}
{"level":"info","ts":"2024-05-31T12:50:58Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":39.29421536056283}
{"level":"info","ts":"2024-05-31T12:51:02Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":39.61190150478796}
{"level":"info","ts":"2024-05-31T12:51:06Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":40.00762491042929}
{"level":"info","ts":"2024-05-31T12:51:10Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":40.16646798254185}
{"level":"info","ts":"2024-05-31T12:51:14Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":40.37881245521464}
{"level":"info","ts":"2024-05-31T12:51:18Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":40.72324929971989}
{"level":"info","ts":"2024-05-31T12:51:22Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":41.09332942479317}
{"level":"info","ts":"2024-05-31T12:51:26Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":41.25217249690574}
{"level":"info","ts":"2024-05-31T12:51:30Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":41.62280633183506}
{"level":"info","ts":"2024-05-31T12:51:34Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":41.86079408507589}
{"level":"info","ts":"2024-05-31T12:51:38Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":41.93993876620416}
{"level":"info","ts":"2024-05-31T12:51:42Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":42.2843756107094}
{"level":"info","ts":"2024-05-31T12:51:46Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":42.65500944563872}
{"level":"info","ts":"2024-05-31T12:51:50Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":42.94649859943977}
{"level":"info","ts":"2024-05-31T12:51:54Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":44.3968308253534}
{"level":"info","ts":"2024-05-31T12:51:58Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":49.5796983909843}
{"level":"info","ts":"2024-05-31T12:52:02Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":49.84443684450524}
{"level":"info","ts":"2024-05-31T12:52:06Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":50.18831997915446}
{"level":"info","ts":"2024-05-31T12:52:10Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":50.47925542309948}
{"level":"info","ts":"2024-05-31T12:52:14Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":50.82369226760473}
{"level":"info","ts":"2024-05-31T12:52:18Z","logger":"cluster","msg":"Rebalancing","cluster":"default/cb-example","progress":51.35427659435868}
{"level":"info","ts":"2024-05-31T12:52:32Z","logger":"cluster","msg":"Reconciliation failed","cluster":"default/cb-example","error":"failed to rebalance: timeout: unexpected rebalance error","stack":"github.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).verifyRebalance.func1\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/rebalance.go:299\ngithub.com/couchbase/couchbase-operator/pkg/util/retryutil.Retry\n\tgithub.com/couchbase/couchbase-operator/pkg/util/retryutil/retryutil.go:14\ngithub.com/couchbase/couchbase-operator/pkg/util/retryutil.RetryFor\n\tgithub.com/couchbase/couchbase-operator/pkg/util/retryutil/retryutil.go:30\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).verifyRebalance\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/rebalance.go:324\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).rebalanceWithRetriesOnVerifyFails\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/rebalance.go:258\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).rebalance\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/rebalance.go:185\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).handleRebalance\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:1527\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).exec\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:319\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcileMembers\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:268\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:177\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).runReconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:492\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).Update\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:535\ngithub.com/couchbase/couchbase-operator/pkg/controller.(*CouchbaseClusterReconciler).Reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/controller/controller.go:90\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:227"}
{"level":"info","ts":"2024-05-31T12:52:32Z","logger":"cluster","msg":"Resource updated","cluster":"default/cb-example","diff":"{v2.ClusterStatus}.Conditions[2].LastUpdateTime:2024-05-31T12:39:58Z->2024-05-31T12:52:32Z;{v2.ClusterStatus}.Conditions[2].Message:reconcile was blocked from running: recovering node cb-example-0001->failed to rebalance: timeout: unexpected rebalance error"}
{"level":"info","ts":"2024-05-31T12:52:32Z","logger":"cluster","msg":"Resource updated","cluster":"default/cb-example","diff":"+{v2.ClusterStatus}.Members.Ready[?->1]:cb-example-0001;-{v2.ClusterStatus}.Members.Ready[3->?]:cb-example-0004;{v2.ClusterStatus}.Members.Unready[0]:cb-example-0001->cb-example-0004"}
{"level":"info","ts":"2024-05-31T12:52:33Z","logger":"cluster","msg":"Cluster status","cluster":"default/cb-example","balance":"unbalanced","rebalancing":false}
{"level":"info","ts":"2024-05-31T12:52:33Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0000","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:33Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0001","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:33Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0002","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:33Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0003","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:33Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0004","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T12:52:33Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0005","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:33Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0006","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:33Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0007","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:33Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0008","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:33Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0009","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:33Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0000","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:52:33Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0001","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:52:33Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0002","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T12:52:33Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0003","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T12:52:33Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0005","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T12:52:33Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0007","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:52:33Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0009","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:52:33Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0006","class":"index-query","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T12:52:33Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0008","class":"index-query","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T12:52:33Z","logger":"cluster","msg":"Resource updated","cluster":"default/cb-example","diff":"-{v2.ClusterStatus}.Conditions[0->?]:{Type:Available Status:True LastUpdateTime:2024-05-31T12:32:03Z LastTransitionTime:2024-05-31T12:32:03Z Reason:Available Message:};+{v2.ClusterStatus}.Conditions[?->0]:{Type:Available Status:False LastUpdateTime:2024-05-31T12:52:33Z LastTransitionTime:2024-05-31T12:52:33Z Reason:PartiallyAvailable Message:The following nodes are down and not serving requests: cb-example-0004}"}
{"level":"info","ts":"2024-05-31T12:52:33Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0004","recovery_in":34.999999244}
{"level":"info","ts":"2024-05-31T12:52:33Z","logger":"cluster","msg":"Reconciliation failed","cluster":"default/cb-example","error":"reconcile was blocked from running: waiting for pod failover","stack":"github.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).handleDownNodes\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:435\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).exec\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:319\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcileMembers\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:268\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:177\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).runReconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:492\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).Update\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:535\ngithub.com/couchbase/couchbase-operator/pkg/controller.(*CouchbaseClusterReconciler).Reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/controller/controller.go:90\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:227"}
{"level":"info","ts":"2024-05-31T12:52:33Z","logger":"cluster","msg":"Resource updated","cluster":"default/cb-example","diff":"{v2.ClusterStatus}.Conditions[2].LastUpdateTime:2024-05-31T12:52:32Z->2024-05-31T12:52:33Z;{v2.ClusterStatus}.Conditions[2].Message:failed to rebalance: timeout: unexpected rebalance error->reconcile was blocked from running: waiting for pod failover"}
{"level":"info","ts":"2024-05-31T12:52:34Z","logger":"cluster","msg":"Cluster status","cluster":"default/cb-example","balance":"unbalanced","rebalancing":false}
{"level":"info","ts":"2024-05-31T12:52:34Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0000","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:34Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0001","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:34Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0002","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:34Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0003","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:34Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0004","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T12:52:34Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0005","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:34Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0006","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:34Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0007","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:34Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0008","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:34Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0009","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:34Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0000","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:52:34Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0001","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:52:34Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0002","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T12:52:34Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0003","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T12:52:34Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0005","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T12:52:34Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0007","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:52:34Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0009","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:52:34Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0006","class":"index-query","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T12:52:34Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0008","class":"index-query","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T12:52:34Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0004","recovery_in":33.775266384}
{"level":"info","ts":"2024-05-31T12:52:34Z","logger":"cluster","msg":"Reconciliation failed","cluster":"default/cb-example","error":"reconcile was blocked from running: waiting for pod failover","stack":"github.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).handleDownNodes\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:435\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).exec\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:319\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcileMembers\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:268\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:177\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).runReconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:492\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).Update\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:535\ngithub.com/couchbase/couchbase-operator/pkg/controller.(*CouchbaseClusterReconciler).Reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/controller/controller.go:90\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:227"}
{"level":"info","ts":"2024-05-31T12:52:42Z","logger":"cluster","msg":"Cluster status","cluster":"default/cb-example","balance":"unbalanced","rebalancing":false}
{"level":"info","ts":"2024-05-31T12:52:42Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0000","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:42Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0001","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:42Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0002","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:42Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0003","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:42Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0004","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T12:52:42Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0005","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:42Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0006","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:42Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0007","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:42Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0008","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:42Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0009","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:42Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0000","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:52:42Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0001","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:52:42Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0002","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T12:52:42Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0003","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T12:52:42Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0005","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T12:52:42Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0007","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:52:42Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0009","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:52:42Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0006","class":"index-query","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T12:52:42Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0008","class":"index-query","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T12:52:42Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0004","recovery_in":25.197937893}
{"level":"info","ts":"2024-05-31T12:52:42Z","logger":"cluster","msg":"Reconciliation failed","cluster":"default/cb-example","error":"reconcile was blocked from running: waiting for pod failover","stack":"github.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).handleDownNodes\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:435\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).exec\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:319\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcileMembers\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:268\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:177\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).runReconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:492\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).Update\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:535\ngithub.com/couchbase/couchbase-operator/pkg/controller.(*CouchbaseClusterReconciler).Reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/controller/controller.go:90\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:227"}
{"level":"info","ts":"2024-05-31T12:52:53Z","logger":"cluster","msg":"Cluster status","cluster":"default/cb-example","balance":"unbalanced","rebalancing":false}
{"level":"info","ts":"2024-05-31T12:52:53Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0000","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:53Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0001","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:53Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0002","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:53Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0003","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:53Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0004","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T12:52:53Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0005","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:53Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0006","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:53Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0007","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:53Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0008","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:53Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0009","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:52:53Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0000","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:52:53Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0001","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:52:53Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0002","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T12:52:53Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0003","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T12:52:53Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0005","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T12:52:53Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0007","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:52:53Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0009","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:52:53Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0006","class":"index-query","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T12:52:53Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0008","class":"index-query","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T12:52:53Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0004","recovery_in":15.040528768}
{"level":"info","ts":"2024-05-31T12:52:53Z","logger":"cluster","msg":"Reconciliation failed","cluster":"default/cb-example","error":"reconcile was blocked from running: waiting for pod failover","stack":"github.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).handleDownNodes\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:435\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).exec\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:319\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcileMembers\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:268\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:177\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).runReconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:492\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).Update\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:535\ngithub.com/couchbase/couchbase-operator/pkg/controller.(*CouchbaseClusterReconciler).Reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/controller/controller.go:90\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:227"}
{"level":"info","ts":"2024-05-31T12:53:03Z","logger":"cluster","msg":"Cluster status","cluster":"default/cb-example","balance":"unbalanced","rebalancing":false}
{"level":"info","ts":"2024-05-31T12:53:03Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0000","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:53:03Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0001","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:53:03Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0002","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:53:03Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0003","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:53:03Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0004","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T12:53:03Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0005","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:53:03Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0006","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:53:03Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0007","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:53:03Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0008","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:53:03Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0009","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:53:03Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0000","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:53:03Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0001","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:53:03Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0002","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T12:53:03Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0003","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T12:53:03Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0005","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T12:53:03Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0007","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:53:03Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0009","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:53:03Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0006","class":"index-query","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T12:53:03Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0008","class":"index-query","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T12:53:03Z","logger":"cluster","msg":"Pod down, waiting for auto-failover","cluster":"default/cb-example","name":"cb-example-0004","recovery_in":4.899988144}
{"level":"info","ts":"2024-05-31T12:53:03Z","logger":"cluster","msg":"Reconciliation failed","cluster":"default/cb-example","error":"reconcile was blocked from running: waiting for pod failover","stack":"github.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).handleDownNodes\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:435\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*ReconcileMachine).exec\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/nodereconcile.go:319\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcileMembers\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:268\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:177\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).runReconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:492\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).Update\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:535\ngithub.com/couchbase/couchbase-operator/pkg/controller.(*CouchbaseClusterReconciler).Reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/controller/controller.go:90\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:227"}
{"level":"info","ts":"2024-05-31T12:53:13Z","logger":"cluster","msg":"Cluster status","cluster":"default/cb-example","balance":"unbalanced","rebalancing":false}
{"level":"info","ts":"2024-05-31T12:53:13Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0000","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:53:13Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0001","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:53:13Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0002","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:53:13Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0003","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:53:13Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0004","version":"7.2.5","class":"data-only","managed":true,"status":"Down"}
{"level":"info","ts":"2024-05-31T12:53:13Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0005","version":"7.2.5","class":"data-only","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:53:13Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0006","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:53:13Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0007","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:53:13Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0008","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:53:13Z","logger":"cluster","msg":"Node status","cluster":"default/cb-example","name":"cb-example-0009","version":"7.2.5","class":"index-query","managed":true,"status":"Active"}
{"level":"info","ts":"2024-05-31T12:53:13Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0000","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:53:13Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0001","class":"data-only","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:53:13Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0002","class":"data-only","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T12:53:13Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0003","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T12:53:13Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0005","class":"data-only","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T12:53:13Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0007","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:53:13Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0009","class":"index-query","group":"us-east-2a"}
{"level":"info","ts":"2024-05-31T12:53:13Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0006","class":"index-query","group":"us-east-2b"}
{"level":"info","ts":"2024-05-31T12:53:13Z","logger":"scheduler","msg":"Scheduler status","cluster":"default/cb-example","name":"cb-example-0008","class":"index-query","group":"us-east-2c"}
{"level":"info","ts":"2024-05-31T12:53:13Z","logger":"cluster","msg":"Node failed","cluster":"default/cb-example","name":"cb-example-0004"}
{"level":"info","ts":"2024-05-31T12:53:13Z","logger":"kubernetes","msg":"Creating pod","cluster":"default/cb-example","name":"cb-example-0004","image":"couchbase/server:7.2.5"}
{"level":"info","ts":"2024-05-31T12:59:02Z","logger":"cluster","msg":"Pod recovering","cluster":"default/cb-example","name":"cb-example-0004"}
{"level":"info","ts":"2024-05-31T12:59:02Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"waiting for cluster recovery"}
{"level":"info","ts":"2024-05-31T12:59:02Z","logger":"cluster","msg":"Resource updated","cluster":"default/cb-example","diff":"-{v2.ClusterStatus}.Conditions[2->?]:{Type:Error Status:True LastUpdateTime:2024-05-31T12:52:33Z LastTransitionTime:2024-05-31T12:39:58Z Reason:ErrorEncountered Message:reconcile was blocked from running: waiting for pod failover}"}
{"level":"info","ts":"2024-05-31T12:59:02Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:59:12Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:59:23Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:59:33Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:59:43Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T12:59:53Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T13:00:03Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T13:00:13Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T13:00:23Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T13:00:33Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T13:00:44Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T13:00:54Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T13:01:04Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T13:01:14Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T13:01:24Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T13:01:34Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T13:01:44Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T13:01:54Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T13:02:04Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T13:02:14Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T13:02:25Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T13:02:35Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T13:02:45Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T13:02:55Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T13:03:05Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T13:03:15Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T13:03:25Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T13:03:35Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T13:03:45Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T13:03:56Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T13:04:06Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T13:04:16Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T13:04:26Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T13:04:36Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T13:04:46Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}
{"level":"info","ts":"2024-05-31T13:04:56Z","logger":"cluster","msg":"Aborting topology reconcile","cluster":"default/cb-example","reason":"pods are warming up"}