[System Test] Indexer panic - panic detected while processing mutation

2024-01-26T12:02:48.240-08:00 [Fatal] plasmaSlice::handleCommandsWorker: panic detected while processing mutation for operation 1 key = <ud>()</ud> docid = <ud>(cbo::0000000e::scope_5.coll_5(histogram)type)</ud> Index idx8_P5ynWX, Bucket bucket2, IndexInstId 12890292103434929746, PartitionId 0

2024-01-26T12:02:48.307-08:00 [Info] rpcServer(v1:0): rpc request:url:/rpc/SyncAndCloseFile args:fh:{id:"/plasma_storage_v1/349964f5c435c19f035a07f92d07be01_ShardToken47_52_e4_f2_d7_94_f2_54/12479605516434530676/shards/shard12479605516434530676/config.json"  gen:1706299368} rsp:err:{errCode:"rpc remote close in progress"} 

2024-01-26T12:02:48.306-08:00 [Fatal] goroutine 56497 [running]:

github.com/couchbase/indexing/secondary/logging.(*destination).StackTraceAll(...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/logging/logging.go:179

github.com/couchbase/indexing/secondary/logging.StackTraceAll()

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/logging/logging.go:319 +0x3c

github.com/couchbase/indexing/secondary/indexer.(*plasmaSlice).handleCommandsWorker.func1()

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:979 +0x2ff

panic({0x17cba80?, 0x2e31180?})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.21.6/go/src/runtime/panic.go:914 +0x21f

github.com/couchbase/plasma.(*freeList).get(0x1e46fefbf09?, 0x2ec5100?, 0xc16521da0e5798c9?)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/wctx.go:1008 +0x3a

github.com/couchbase/plasma.(*Plasma).getFreeWCtx(0xc03702f500, 0x1fa7470?, 0x1)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/wctx.go:622 +0x65

github.com/couchbase/plasma.(*Plasma).getWCtx(0xc03702f500, 0x442a5f?, 0x1, 0x0?)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/wctx.go:555 +0x6f

github.com/couchbase/plasma.(*Plasma).newWCtxForWorker(...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/wctx.go:547

github.com/couchbase/plasma.(*Writer).initCtx(...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/writer.go:218

github.com/couchbase/plasma.(*Writer).TryThrottle(0xc01a548900)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/writer.go:37 +0x35

github.com/couchbase/indexing/secondary/indexer.(*plasmaSlice).deleteSecArrayIndex(0xc02a6a5200, {0xc05e269290, 0x2c, 0x30}, 0x5, 0xff?)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:1726 +0xb2

github.com/couchbase/indexing/secondary/indexer.(*plasmaSlice).delete2(0xc02a6a5200, {0xc05e269290?, 0xc02dac2edc?, 0xc0027dd040?}, 0xc02dac2f28?, 0x40?)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:1613 +0x30

github.com/couchbase/indexing/secondary/indexer.(*plasmaSlice).insert(0xc02a6a5200, {0x0?, 0x0?, 0xc01871f140?}, {0xc05e269290?, 0xc01871f15c?, 0xc01871f140?}, 0x1fa2220?, 0x10?, 0xc0327a2700)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:1115 +0xbf

github.com/couchbase/indexing/secondary/indexer.(*plasmaSlice).handleCommandsWorker(0xc02a6a5200, 0x5)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:993 +0x41d

created by github.com/couchbase/indexing/secondary/indexer.(*plasmaSlice).initWriters in goroutine 9666

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:4444 +0x3ef