Query service is crashing with unexpected fault address, segmentation violation error during system test run

2024-01-26T06:56:20.589Z, ns_log:0:info:message(ns_1@svc-q-node-006.ueinmg2zt81jqib2.sandbox.nonprod-project-avengers.com) - Service 'n1ql' exited with status 2. Restarting. Messages:

github.com/couchbase/gometa/protocol.RunWatcherServerWithRequest2({0xc001466a00, 0x49}, {0x3a06c80, 0xc001d6db00}, {0x3a28168, 0xc001d6db00}, {0x3a23380, 0x53f8d40}, 0xc0068092d0, 0xc000e51740, ...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gometa/protocol/watcherServer.go:65 +0x165 fp=0xc006389f68 sp=0xc006389e90 pc=0x28ec605

github.com/couchbase/indexing/secondary/manager/client.(*MetadataProvider).startWatcher.func1()

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/manager/client/metadata_provider.go:4515 +0x65 fp=0xc006389fe0 sp=0xc006389f68 pc=0x295a925

runtime.goexit()

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.21.6/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc006389fe8 sp=0xc006389fe0 pc=0x123cda1

created by github.com/couchbase/indexing/secondary/manager/client.(*MetadataProvider).startWatcher in goroutine 2575

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/manager/client/metadata_provider.go:4515 +0x1cf

unexpected fault address 0x0

fatal error: fault

[signal SIGSEGV: segmentation violation code=0x80 addr=0x0 pc=0x2a9d28b]

goroutine 16256617 [running]:

runtime.throw({0x30bfae2?, 0xc164f3c516e997cb?})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.21.6/go/src/runtime/panic.go:1077 +0x5c fp=0xc008631500 sp=0xc0086314d0 pc=0x1204a5c

runtime.sigpanic()

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.21.6/go/src/runtime/signal_unix.go:875 +0x285 fp=0xc008631560 sp=0xc008631500 pc=0x121ce05

github.com/couchbase/query/tenant.NeedRefund(...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/tenant/tenant_ee.go:274

github.com/couchbase/query/server/http.(*httpRequest).Execute(0xc001be4000, 0x3a3c0b8?, 0xc001504a80, {0x30c158b, 0x6}, {0x3a346b8, 0xc00660e080}, 0x0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/server/http/service_response.go:276 +0x50b fp=0xc008631670 sp=0xc008631560 pc=0x2a9d28b

github.com/couchbase/query/server.(*Server).serviceRequest(0xc000b0e000, {0x3abaeb8?, 0xc001be4000})

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/server/server.go:1419 +0x11b1 fp=0xc008631810 sp=0xc008631670 pc=0x2a144b1

github.com/couchbase/query/server.(*Server).handlePlusRequest(0xc000b0e000?, {0x3abaeb8, 0xc001be4000}, 0x2a09bf5?, 0xc008631880?)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/server/server.go:872 +0x211 fp=0xc008631870 sp=0xc008631810 pc=0x2a11a11

github.com/couchbase/query/server.(*Server).PlusServiceRequest(0xc000b0e000, {0x3abaeb8, 0xc001be4000})

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/server/server.go:753 +0x53 fp=0xc0086318a8 sp=0xc008631870 pc=0x2a109b3

github.com/couchbase/query/server/http.(*HttpEndpoint).ServeHTTP(0xc000bb4680, {0x3a063e0, 0xc00309cfc0}, 0xc0016ad500)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/server/http/service_endpoint.go:395 +0xa6b fp=0xc0086319f8 sp=0xc0086318a8 pc=0x2a8e9cb

github.com/couchbase/query/server/http.(*HttpEndpoint).ServeHTTP-fm({0x3a063e0?, 0xc00309cfc0?}, 0xc008631a88?)

	<autogenerated>:1 +0x36 fp=0xc008631a28 sp=0xc0086319f8 pc=0x2ab8136

github.com/couchbase/query/server/http/router.(*routerImpl).ServeHTTP(0xc00047b140, {0x3a063e0, 0xc00309cfc0}, 0xc0016ad400)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/server/http/router/router.go:217 +0x204 fp=0xc008631b48 sp=0xc008631a28 pc=0x172cfa4

net/http.serverHandler.ServeHTTP({0x3a029d0?}, {0x3a063e0?, 0xc00309cfc0?}, 0x6?)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.21.6/go/src/net/http/server.go:2938 +0x8e fp=0xc008631b78 sp=0xc008631b48 pc=0x1585eee

net/http.(*conn).serve(0xc000158ea0, {0x3a09790, 0xc000e18960})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.21.6/go/src/net/http/server.go:2009 +0x5f4 fp=0xc008631fb8 sp=0xc008631b78 pc=0x1581ab4

net/http.(*Server).Serve.func3()

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.21.6/go/src/net/http/server.go:3086 +0x28 fp=0xc008631fe0 sp=0xc008631fb8 pc=0x15867c8

runtime.goexit()

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.21.6/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc008631fe8 sp=0xc008631fe0 pc=0x123cda1

created by net/http.(*Server).Serve in goroutine 335

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.21.6/go/src/net/http/server.go:3086 +0x5cb