Rebalance failure because of indexer panic - invalid memory address or nil pointer dereference

2023-11-16T17:48:34.612+05:30 [Info] switched currmeta from 1322 -> 1322 force true 

2023-11-16T17:48:34.630+05:30 [Info] lifecycleMgr.dispatchRequest: op OPCODE_UPDATE_INDEX_INST elapsed 397.61095ms len(expediates) 0 len(incomings) 1 len(outgoings) 0 len(parallels) 0 error <nil>

2023-11-16T17:48:34.630+05:30 [Info] lifecycleMgr.dispatchRequest: op OPCODE_ASYNC_RECOVERY_DONE elapsed 140.936µs len(expediates) 0 len(incomings) 0 len(outgoings) 1 len(parallels) 0 error <nil>

2023-11-16T17:48:34.671+05:30 [Info] ShardRebalancer::waitForIndexState: Indexes: map[8691999751317018172:INDEX_STATE_RECOVERED] reached state: INDEX_STATE_RECOVERED

2023-11-16T17:48:34.685+05:30 [Info] Shard /opt/couchbase/var/lib/couchbase/data/@2i/shards/shard6835955389805966770(6835955389805966770)(742617059685901209-1-1) : Shard.doRecovery: Done recovery. Recovery log [/opt/couchbase/var/lib/couchbase/data/@2i/test_bucket_hotelcd6ec9a9547b4a02884e239cc6494529array_index_rooms_11990368271953021114_0.index/docIndex/recovery], Data log [/opt/couchbase/var/lib/couchbase/data/@2i/test_bucket_hotelcd6ec9a9547b4a02884e239cc6494529array_index_rooms_11990368271953021114_0.index/docIndex], Shared [false]. Built [1] plasmas, took [575.036974ms]

panic: runtime error: invalid memory address or nil pointer dereference

[signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x121eef1]

goroutine 1008268 [running]:

github.com/couchbase/plasma.(*sCtx).holdLSS(...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/smr.go:69

github.com/couchbase/plasma.(*wCtx).BeginTx(0xc0159ec829?)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/smr.go:82 +0x11

github.com/couchbase/plasma.(*Plasma).tryEvictPagesOnce(0xc01b7aa000, 0xc01430f680, 0x1, 0x0, 0xe5?)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/swapper.go:450 +0xeb

github.com/couchbase/plasma.evictPlasma.func1(0xc020fb8b80, {0xc00ec1b8b0?, 0x10db9?, 0x2e5d080?})

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/recovery.go:2584 +0x2f9

github.com/couchbase/plasma.evictPlasma(0xc005661ba0, 0xc017aea480)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/recovery.go:2617 +0x13a

github.com/couchbase/plasma.(*Shard).recoverFromDataReplay.func3(0xc014ec41e0?, {0x7f46e335067e, 0x9fe, 0x98982}, 0x0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/recovery.go:1185 +0x2db

github.com/couchbase/plasma.(*lsStore).Visitor2(0xc00c4ca400?, 0x1a1eb70?, 0x8a2000, 0xc00c273f90, 0xc0171de280)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/lss.go:598 +0x10d

github.com/couchbase/plasma.(*LSSCtx).GetLSSVisitor(...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/lssctx.go:447

github.com/couchbase/plasma.(*Shard).recoverFromDataReplay(0xc017aea480, 0xc005661ba0, 0xc01fd36000, 0x551716, 0xc01b1f65f0, 0xc0152ec6c8, 0xc0003e9780)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/recovery.go:1201 +0x597

github.com/couchbase/plasma.(*Shard).doRecovery(0xc017aea480, 0xc005661ba0, 0xc01fd36000, 0xc01fd36780, 0xc00be34300, 0x0, 0x0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/recovery.go:420 +0xfb3

github.com/couchbase/plasma.(*Shard).recoverFromShard.func2()

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/recovery.go:228 +0x3e

github.com/couchbase/plasma.(*Shard).recoverFromShard(0xc017aea480, 0x0?, 0x0?, 0x0?, 0x0?, 0x1?, 0x0?)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/recovery.go:247 +0x105

github.com/couchbase/plasma.doRecovery({0x0?, 0x1e?}, 0x12c?, 0xc00be34300, 0xc017aea480, 0xc01fd36000, 0x1b09fd0?, 0x1b0a018?)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/recovery.go:2152 +0xb7

github.com/couchbase/plasma.New6({_, _}, {_, _}, {{0x0, 0x0}, 0x1e, 0x12c, 0x5, 0x4, ...}, ...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/plasma.go:466 +0x4bf

github.com/couchbase/indexing/secondary/indexer.(*plasmaSlice).initStores.func3()

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:618 +0x39a

created by github.com/couchbase/indexing/secondary/indexer.(*plasmaSlice).initStores in goroutine 1008265

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:608 +0x32f

2023-11-16T17:48:35.486+05:30 [Info] Indexer started with command line: [/opt/couchbase/bin/indexer -adminPort=9100 -scanPort=9101 -httpPort=9102 -streamInitPort=9103 -streamCatchupPort=9104 -streamMaintPort=9105 --httpsPort=19102 --certFile=/opt/couchbase/var/lib/couchbase/config/certs/chain.pem --keyFile=/opt/couchbase/var/lib/couchbase/config/certs/pkey.pem --caFile=/opt/couchbase/var/lib/couchbase/config/certs/ca.pem --clientCertFile=/opt/couchbase/var/lib/couchbase/config/certs/client_chain.pem --clientKeyFile=/opt/couchbase/var/lib/couchbase/config/certs/client_pkey.pem -ipv4=required -ipv6=optional -cluster=127.0.0.1:8091 -storageDir=/opt/couchbase/var/lib/couchbase/data/@2i -diagDir=/opt/couchbase/var/lib/couchbase/crash -logDir=/opt/couchbase/var/lib/couchbase/logs