[System Test] Plasma panic : fatal: Incorrect page split detected - splitKey:item key

-test tests/integration/neo/test_neo.yml -scope tests/integration/neo/scope_couchstore.yml

2023-02-06T16:13:54.369-08:00, ns_log:0:info:message(ns_1@172.23.120.245) - Service 'index' exited with status 2. Restarting. Messages:

github.com/couchbase/indexing/secondary/indexer.(*plasmaSlice).insertSecArrayIndex(0xc01512a000, {0xc02587a000, 0x3d9, 0x400}, {0xc02115d130, 0x8, 0x8}, 0x6, 0x0, 0xc0151adec0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:1084 +0x2213

github.com/couchbase/indexing/secondary/indexer.(*plasmaSlice).insert(0xc01512a000, {0xc02587a000?, 0x0?, 0x2?}, {0xc02115d130?, 0x9?, 0x10?}, 0x6?, 0x0?, 0xc0151adec0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:805 +0xa6

github.com/couchbase/indexing/secondary/indexer.(*plasmaSlice).handleCommandsWorker(0xc01512a000, 0x6)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:680 +0x425

created by github.com/couchbase/indexing/secondary/indexer.(*plasmaSlice).initWriters

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:3199 +0x3cd

2023-02-06T16:14:02.441-08:00, indexing:0:info:message(ns_1@172.23.120.245) - Detected storage corruption for index idx10_HjdZLJAJ, partition id 0. Starting cleanup.

panic: bucket6/idx10_HjdZLJAJ/Mainstore#12329215668393975940:0 : fatal: Incorrect page split detected - splitKey:item key: ...truncated due to encoding error

2023-02-06T16:13:53.898-08:00 [Fatal] goroutine 33485740 [running]:

github.com/couchbase/indexing/secondary/logging.(*destination).StackTraceAll(...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/logging/logging.go:175

github.com/couchbase/indexing/secondary/logging.StackTraceAll()

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/logging/logging.go:317 +0x3f

github.com/couchbase/indexing/secondary/indexer.(*plasmaSlice).handleCommandsWorker.func1()

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:666 +0x30f

panic({0x1031ea0, 0xc03599c6e0})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/runtime/panic.go:838 +0x207

github.com/couchbase/plasma.(*allocCtx).Reset(0xc011dfce00?)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/page_alloc.go:150 +0x146

github.com/couchbase/plasma.(*Plasma).freeWCtx(0xc011dfce00, 0x173d, 0xc02fa20bd0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/wctx.go:578 +0x74

github.com/couchbase/plasma.(*Writer).freeCtx(0xc01a69be80)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/writer.go:199 +0x47

github.com/couchbase/plasma.(*Writer).End(0xc01a69be80)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/writer.go:58 +0x92

panic({0x1068ea0, 0xc03599c5e0})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/runtime/panic.go:844 +0x258

github.com/couchbase/plasma.(*Shard).raisePanic(...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/shard.go:1051

github.com/couchbase/plasma.(*Plasma).fatalPanic(0xc011dfce00, {0xc02c1b7b80, 0x144})

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/plasma.go:1645 +0x316

github.com/couchbase/plasma.(*Plasma).trySMOs2(0xc011dfce00, {0xc02fa20bd0?}, {0x1450d38, 0xc002650e00}, 0xc02fa20bd0, 0x1, 0x200?, 0x600?, 0x702?, 0x4)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/plasma.go:1400 +0x578

github.com/couchbase/plasma.(*Plasma).trySMOs(...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/plasma.go:1352

github.com/couchbase/plasma.(*Writer).Insert(0xc01a69be80, 0x0?)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/writer.go:82 +0x145

github.com/couchbase/plasma.(*Writer).InsertKV(0xc01a69be80, {0xc00ae58000, 0x1d, 0x702}, {0x0, 0x0, 0x0})

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/mvcc.go:471 +0x225

github.com/couchbase/indexing/secondary/indexer.(*plasmaSlice).insertSecArrayIndex(0xc01512a000, {0xc02587a000, 0x3d9, 0x400}, {0xc02115d130, 0x8, 0x8}, 0x6, 0x0, 0xc0151adec0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:1084 +0x2213

github.com/couchbase/indexing/secondary/indexer.(*plasmaSlice).insert(0xc01512a000, {0xc02587a000?, 0x0?, 0x2?}, {0xc02115d130?, 0x9?, 0x10?}, 0x6?, 0x0?, 0xc0151adec0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:805 +0xa6

github.com/couchbase/indexing/secondary/indexer.(*plasmaSlice).handleCommandsWorker(0xc01512a000, 0x6)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:680 +0x425

created by github.com/couchbase/indexing/secondary/indexer.(*plasmaSlice).initWriters

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:3199 +0x3cd

goroutine 1 [select]:

github.com/couchbase/indexing/secondary/indexer.(*indexer).run(0xc000258700)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:1065 +0x1f9

github.com/couchbase/indexing/secondary/indexer.NewIndexer(0x2003560?)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:511 +0x262c

main.main()

	goproj/src/github.com/couchbase/indexing/secondary/cmd/indexer/main.go:182 +0xfae