Details
-
Bug
-
Resolution: Fixed
-
Major
-
None
-
None
-
None
-
5 - Forces of Destiny, 7 -The Rebellion Begins
-
1
Description
The failure is a scheme failure, but there is a suspicious-looking error in the logs:
Error MessageEvent schema validation failed:
|
| NewMemberAdded | New member test-couchbase-2wgkl-0000 added to cluster | |
| NewMemberAdded | New member test-couchbase-2wgkl-0001 added to cluster | |
| NewMemberAdded | New member test-couchbase-2wgkl-0002 added to cluster | |
| RebalanceStarted | A rebalance has been started to balance data across the cluster |
|
| RebalanceCompleted | A rebalance has completed |
|
| NewMemberAdded | New member test-couchbase-2wgkl-0003 added to cluster | |
| RebalanceStarted | A rebalance has been started to balance data across the cluster |
|
| MemberRemoved | Existing member test-couchbase-2wgkl-0000 removed from the cluster | |
| RebalanceCompleted | A rebalance has completed | <== schema underflowed event stream
|
Error formatting macro: code: java.lang.StackOverflowError
{"level":"info","ts":"2024-03-07T09:20:55Z","logger":"cluster","msg":"Reconciliation failed","cluster":"test-5m9l8/test-couchbase-2wgkl","error":"unexpected status code: request failed GET http://test-couchbase-2wgkl-0000.test-couchbase-2wgkl.test-5m9l8.svc:8091/pools/default 500 Internal Server Error: [\"Unexpected server error, request logged.\"]","stack":"github.com/couchbase/couchbase-operator/pkg/util/couchbaseutil.Client.doRequest\n\tgithub.com/couchbase/couchbase-operator/pkg/util/couchbaseutil/core.go:212\ngithub.com/couchbase/couchbase-operator/pkg/util/couchbaseutil.(*Client).Get\n\tgithub.com/couchbase/couchbase-operator/pkg/util/couchbaseutil/core.go:261\ngithub.com/couchbase/couchbase-operator/pkg/util/couchbaseutil.(*Request).On.func1\n\tgithub.com/couchbase/couchbase-operator/pkg/util/couchbaseutil/api.go:222\ngithub.com/couchbase/couchbase-operator/pkg/util/couchbaseutil.(*Request).On\n\tgithub.com/couchbase/couchbase-operator/pkg/util/couchbaseutil/api.go:249\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).getStatus\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/status.go:112\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).GetStatusUnsafe\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/status.go:105\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).updateMembers\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/member.go:41\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).create\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:382\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/reconcile.go:153\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).runReconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:506\ngithub.com/couchbase/couchbase-operator/pkg/cluster.New\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:183\ngithub.com/couchbase/couchbase-operator/pkg/controller.(*CouchbaseClusterReconciler).Reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/controller/controller.go:74\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:227"}
{"level":"info","ts":"2024-03-07T09:20:55Z","logger":"cluster","msg":"Resource updated","cluster":"test-5m9l8/test-couchbase-2wgkl","diff":"+
{v2.ClusterStatus}.Conditions[?->1]:
{Type:Error Status:True LastUpdateTime:2024-03-07T09:20:55Z LastTransitionTime:2024-03-07T09:20:55Z Reason:ErrorEncountered Message:unexpected status code: request failed GET http://test-couchbase-2wgkl-0000.test-couchbase-2wgkl.test-5m9l8.svc:8091/pools/default 500 Internal Server Error: [\"Unexpected server error, request logged.\"]}"}
{"level":"debug","ts":"2024-03-07T09:20:59Z","logger":"api","msg":"http","cluster":"test-5m9l8/test-couchbase-2wgkl","method":"GET","url":"http://test-couchbase-2wgkl-0000.test-couchbase-2wgkl.test-5m9l8.svc:8091/pools/default","status":"500 Internal Server Error","time_ms":3937.535324} {"level":"error","ts":"2024-03-07T09:20:59Z","logger":"cluster","msg":"Failed to update members","cluster":"test-5m9l8/test-couchbase-2wgkl","error":"unexpected status code: request failed GET http://test-couchbase-2wgkl-0000.test-couchbase-2wgkl.test-5m9l8.svc:8091/pools/default 500 Internal Server Error: [\"Unexpected server error, request logged.\"]","stacktrace":"github.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).runReconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:468\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).Update\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:549\ngithub.com/couchbase/couchbase-operator/pkg/controller.(*CouchbaseClusterReconciler).Reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/controller/controller.go:90\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:227"} {"level":"error","ts":"2024-03-07T09:20:59Z","logger":"cluster","msg":"Failed to rotate expired certificates","cluster":"test-5m9l8/test-couchbase-2wgkl","error":"TLS invalid: Attempted to check if certifiates are expired but TLS was never initialized","stacktrace":"github.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).runReconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:493\ngithub.com/couchbase/couchbase-operator/pkg/cluster.(*Cluster).Update\n\tgithub.com/couchbase/couchbase-operator/pkg/cluster/cluster.go:549\ngithub.com/couchbase/couchbase-operator/pkg/controller.(*CouchbaseClusterReconciler).Reconcile\n\tgithub.com/couchbase/couchbase-operator/pkg/controller/controller.go:90\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\tsigs.k8s.io/controller-runtime@v0.16.3/pkg/internal/controller/controller.go:227"} {"level":"debug","ts":"2024-03-07T09:21:05Z","logger":"api","msg":"http","cluster":"test-5m9l8/test-couchbase-2wgkl","method":"GET","url":"http://test-couchbase-2wgkl-0000.test-couchbase-2wgkl.test-5m9l8.svc:8091/pools/default","status":"200 OK","time_ms":8.275883}