Rebalance failure due to indexer panic - invalid memory address or nil pointer dereference

2023-11-18T06:34:23.784-08:00 [Info] Shard /opt/couchbase/var/lib/couchbase/data/@2i/shards/shard12243813062001242407(12243813062001242407)(5843534361635924470-1-0) : Shard.doRecovery: Done recovery. Recovery log [/opt/couchbase/var/lib/couchbase/data/@2i/test_bucket_hotel211a60ce06f94579b5735b09650859fapartitioned_index_11292587679287037249_7.index/mainIndex/recovery], Data log [/opt/couchbase/var/lib/couchbase/data/@2i/test_bucket_hotel211a60ce06f94579b5735b09650859fapartitioned_index_11292587679287037249_7.index/mainIndex], Shared [false]. Built [1] plasmas, took [1.194393367s]

panic: runtime error: invalid memory address or nil pointer dereference

[signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x121f471]

goroutine 1985461 [running]:

github.com/couchbase/plasma.(*sCtx).holdLSS(...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/smr.go:69

github.com/couchbase/plasma.(*wCtx).BeginTx(0xc017257248?)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/smr.go:82 +0x11

github.com/couchbase/plasma.(*Plasma).tryEvictPagesOnce(0xc0082fa000, 0xc01f9c0500, 0x1, 0x0, 0xe5?)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/swapper.go:450 +0xeb

github.com/couchbase/plasma.evictPlasma.func1(0xc00acb6b00, {0xc0116aa500?, 0x4a?, 0x2e5f0a0?})

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/recovery.go:2586 +0x2f9

github.com/couchbase/plasma.evictPlasma(0xc005fe3040, 0xc006c54180)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/recovery.go:2619 +0x13a

github.com/couchbase/plasma.(*Shard).recoverFromDataReplay.func3(0xc0151360f0?, {0x7fae3c2cbde3, 0x1632, 0xad21d}, 0x0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/recovery.go:1187 +0x2db

github.com/couchbase/plasma.(*lsStore).Visitor2(0xc01b22c700?, 0x1a1fd6f?, 0x8f5000, 0xc00c0a9090, 0xc0180f0700)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/lss.go:598 +0x10d

github.com/couchbase/plasma.(*LSSCtx).GetLSSVisitor(...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/lssctx.go:447

github.com/couchbase/plasma.(*Shard).recoverFromDataReplay(0xc006c54180, 0xc005fe3040, 0xc008105a40, 0x2e4000, 0xc007c88af0, 0xc0174446c8, 0xc016a72590)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/recovery.go:1203 +0x597

github.com/couchbase/plasma.(*Shard).doRecovery(0xc006c54180, 0xc005fe3040, 0xc008105a40, 0xc008105b80, 0xc017166000, 0x0, 0x0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/recovery.go:421 +0xfb3

github.com/couchbase/plasma.(*Shard).recoverFromShard.func2()

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/recovery.go:228 +0x3e

github.com/couchbase/plasma.(*Shard).recoverFromShard(0xc006c54180, 0x0?, 0x0?, 0x0?, 0x0?, 0x1?, 0x0?)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/recovery.go:247 +0x105

github.com/couchbase/plasma.doRecovery({0x0?, 0xc8?}, 0x190?, 0xc017166000, 0xc006c54180, 0xc008105a40, 0x1b0b3b0?, 0x1b0b3f8?)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/recovery.go:2154 +0xb7

github.com/couchbase/plasma.New6({_, _}, {_, _}, {{0x0, 0x0}, 0xc8, 0x190, 0x19, 0x4, ...}, ...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/plasma.go:466 +0x4bf

github.com/couchbase/indexing/secondary/indexer.(*plasmaSlice).initStores.func2()

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:598 +0x38f

created by github.com/couchbase/indexing/secondary/indexer.(*plasmaSlice).initStores in goroutine 1983556

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:588 +0x272

2023-11-18T06:34:24.097-08:00 [Info] Indexer started with command line: [/opt/couchbase/bin/indexer -adminPort=9100 -scanPort=9101 -httpPort=9102 -streamInitPort=9103 -streamCatchupPort=9104 -streamMaintPort=9105 --httpsPort=19102 --certFile=/opt/couchbase/var/lib/couchbase/config/certs/chain.pem --keyFile=/opt/couchbase/var/lib/couchbase/config/certs/pkey.pem --caFile=/opt/couchbase/var/lib/couchbase/config/certs/ca.pem --clientCertFile=/opt/couchbase/var/lib/couchbase/config/certs/client_chain.pem --clientKeyFile=/opt/couchbase/var/lib/couchbase/config/certs/client_pkey.pem -ipv4=required -ipv6=optional -cluster=127.0.0.1:8091 -storageDir=/opt/couchbase/var/lib/couchbase/data/@2i -diagDir=/opt/couchbase/var/lib/couchbase/crash -logDir=/opt/couchbase/var/lib/couchbase/logs