[System test] Panic observed in query logs

-test tests/integration/cheshirecat/test_cheshirecat_kv_gsi_coll_xdcr_backup_sgw_fts_itemct_txns_eventing_cbas_scale3.yml -scope tests/integration/cheshirecat/scope_cheshirecat_with_backup.yml

2021-09-07T17:56:34.265-07:00 [SEVERE] panic: Cannot create value for type *errors.errorString.

2021-09-07T17:56:34.265-07:00 [SEVERE] request text: <ud>select * from system:completed_requests where errors[0].message = "DML Error, possible causes include concurrent modification Failed to perform UPSERT" limit 1</ud>

2021-09-07T17:56:34.265-07:00 [SEVERE] query context: <ud></ud>

2021-09-07T17:56:34.265-07:00 [SEVERE] stack: goroutine 82108711 [running]:

github.com/couchbase/query/execution.(*Context).Recover(0xc0f9609b80, 0xc10f2d3440)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/context.go:985 +0xc6

panic(0x20f78c0, 0xc0acf54120)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.16.6/go/src/runtime/panic.go:971 +0x499

github.com/couchbase/query/execution.(*base).close(0xc10f2d3440, 0xc0f9609b80)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/base.go:273 +0x233

panic(0x20f78c0, 0xc0acf54120)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.16.6/go/src/runtime/panic.go:971 +0x499

github.com/couchbase/query/execution.(*base).notify(0xc10f2d3440)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/base.go:988 +0x178

panic(0x20f78c0, 0xc0acf54120)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.16.6/go/src/runtime/panic.go:971 +0x499

github.com/couchbase/query/value.NewValue(0x214de00, 0xc1664f8060, 0x245b325, 0x5)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/value/value.go:449 +0xd37

github.com/couchbase/query/value.objectValue.WriteJSON(0xc00bf4dfb0, 0x268bd40, 0xc15e088f60, 0x0, 0x0, 0x0, 0x0, 0xc00bf48701, 0x0, 0x0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/value/object.go:175 +0x2ec

github.com/couchbase/query/value.sliceValue.WriteJSON(0xc0acf54100, 0x1, 0x1, 0x268bd40, 0xc15e088f60, 0x0, 0x0, 0x0, 0x0, 0x2458d01, ...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/value/array.go:74 +0x3ef

github.com/couchbase/query/value.objectValue.WriteJSON(0xc00bf4df80, 0x268bd40, 0xc15e088f60, 0x0, 0x0, 0x0, 0x0, 0x2458d01, 0x0, 0x0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/value/object.go:215 +0x687

github.com/couchbase/query/value.(*annotatedValue).WriteJSON(0xc00c6c3c20, 0x268bd40, 0xc15e088f60, 0x0, 0x0, 0x0, 0x0, 0xc14f47a001, 0x24dd838, 0xc14f47a000)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/value/annotated.go:136 +0x8b

github.com/couchbase/query/value.objectValue.WriteJSON(0xc00bf4d7a0, 0x268bd40, 0xc15e088f60, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/value/object.go:155 +0xcb8

github.com/couchbase/query/value.(*ScopeValue).WriteJSON(0xc00bf4d7d0, 0x268bd40, 0xc15e088f60, 0x0, 0x0, 0x0, 0x0, 0xc00c693b01, 0x0, 0x0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/value/scope.go:87 +0x11b

github.com/couchbase/query/value.(*annotatedValue).WriteJSON(0xc00bf3b560, 0x268bd40, 0xc15e088f60, 0x0, 0x0, 0x0, 0x0, 0xc11a82ba01, 0x38094c0, 0x38094c0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/value/annotated.go:136 +0x8b

github.com/couchbase/query/server/http.(*httpRequest).Result(0xc0c4549000, 0x271ad80, 0xc00bf3b560, 0xc10f2d3440)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/server/http/service_response.go:268 +0x20a

github.com/couchbase/query/execution.setup(0xc0f9609b80, 0x271ad80, 0xc00bf3b560, 0xc00bf3b560)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/context.go:556 +0x8d

github.com/couchbase/query/execution.(*Context).Result(...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/context.go:564

github.com/couchbase/query/execution.(*Stream).processItem(0xc10f2d3440, 0x271ad80, 0xc00bf3b560, 0xc0f9609b80, 0xc00bf3b560)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/stream.go:70 +0x4a

github.com/couchbase/query/execution.(*base).runConsumer.func1()

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/base.go:915 +0x2d5

github.com/couchbase/query/util.(*Once).Do(0xc10f2d3578, 0xc00c693f00)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/util/sync.go:55 +0x4b

github.com/couchbase/query/execution.(*base).runConsumer(0xc10f2d3440, 0x26b4d68, 0xc10f2d3440, 0xc0f9609b80, 0x0, 0x0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/base.go:857 +0xa5

github.com/couchbase/query/execution.(*Stream).RunOnce(0xc10f2d3440, 0xc0f9609b80, 0x0, 0x0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/stream.go:66 +0x5c

github.com/couchbase/query/execution.execOp(0x26e2868, 0xc10f2d3440, 0xc0f9609b80, 0x0, 0x0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/base.go:522 +0x57

created by github.com/couchbase/query/execution.(*base).fork

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/base.go:533 +0x105

goroutine 82108711 [running]:

github.com/couchbase/query/execution.(*Context).Recover(0xc0f9609b80, 0xc10f2d3440)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/context.go:985 +0xc6

panic(0x20f78c0, 0xc0acf54120)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.16.6/go/src/runtime/panic.go:971 +0x499

github.com/couchbase/query/execution.(*base).close(0xc10f2d3440, 0xc0f9609b80)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/base.go:273 +0x233

panic(0x20f78c0, 0xc0acf54120)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.16.6/go/src/runtime/panic.go:971 +0x499

github.com/couchbase/query/execution.(*base).notify(0xc10f2d3440)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/base.go:988 +0x178

panic(0x20f78c0, 0xc0acf54120)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.16.6/go/src/runtime/panic.go:971 +0x499

github.com/couchbase/query/value.NewValue(0x214de00, 0xc1664f8060, 0x245b325, 0x5)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/value/value.go:449 +0xd37

github.com/couchbase/query/value.objectValue.WriteJSON(0xc00bf4dfb0, 0x268bd40, 0xc15e088f60, 0x0, 0x0, 0x0, 0x0, 0xc00bf48701, 0x0, 0x0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/value/object.go:175 +0x2ec

github.com/couchbase/query/value.sliceValue.WriteJSON(0xc0acf54100, 0x1, 0x1, 0x268bd40, 0xc15e088f60, 0x0, 0x0, 0x0, 0x0, 0x2458d01, ...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/value/array.go:74 +0x3ef

github.com/couchbase/query/value.objectValue.WriteJSON(0xc00bf4df80, 0x268bd40, 0xc15e088f60, 0x0, 0x0, 0x0, 0x0, 0x2458d01, 0x0, 0x0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/value/object.go:215 +0x687

github.com/couchbase/query/value.(*annotatedValue).WriteJSON(0xc00c6c3c20, 0x268bd40, 0xc15e088f60, 0x0, 0x0, 0x0, 0x0, 0xc14f47a001, 0x24dd838, 0xc14f47a000)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/value/annotated.go:136 +0x8b

github.com/couchbase/query/value.objectValue.WriteJSON(0xc00bf4d7a0, 0x268bd40, 0xc15e088f60, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/value/object.go:155 +0xcb8

github.com/couchbase/query/value.(*ScopeValue).WriteJSON(0xc00bf4d7d0, 0x268bd40, 0xc15e088f60, 0x0, 0x0, 0x0, 0x0, 0xc00c693b01, 0x0, 0x0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/value/scope.go:87 +0x11b

github.com/couchbase/query/value.(*annotatedValue).WriteJSON(0xc00bf3b560, 0x268bd40, 0xc15e088f60, 0x0, 0x0, 0x0, 0x0, 0xc11a82ba01, 0x38094c0, 0x38094c0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/value/annotated.go:136 +0x8b

github.com/couchbase/query/server/http.(*httpRequest).Result(0xc0c4549000, 0x271ad80, 0xc00bf3b560, 0xc10f2d3440)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/server/http/service_response.go:268 +0x20a

github.com/couchbase/query/execution.setup(0xc0f9609b80, 0x271ad80, 0xc00bf3b560, 0xc00bf3b560)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/context.go:556 +0x8d

github.com/couchbase/query/execution.(*Context).Result(...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/context.go:564

github.com/couchbase/query/execution.(*Stream).processItem(0xc10f2d3440, 0x271ad80, 0xc00bf3b560, 0xc0f9609b80, 0xc00bf3b560)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/stream.go:70 +0x4a

github.com/couchbase/query/execution.(*base).runConsumer.func1()

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/base.go:915 +0x2d5

github.com/couchbase/query/util.(*Once).Do(0xc10f2d3578, 0xc00c693f00)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/util/sync.go:55 +0x4b

github.com/couchbase/query/execution.(*base).runConsumer(0xc10f2d3440, 0x26b4d68, 0xc10f2d3440, 0xc0f9609b80, 0x0, 0x0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/base.go:857 +0xa5

github.com/couchbase/query/execution.(*Stream).RunOnce(0xc10f2d3440, 0xc0f9609b80, 0x0, 0x0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/stream.go:66 +0x5c

github.com/couchbase/query/execution.execOp(0x26e2868, 0xc10f2d3440, 0xc0f9609b80, 0x0, 0x0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/base.go:522 +0x57

created by github.com/couchbase/query/execution.(*base).fork

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/base.go:533 +0x105