[Stress Test] Indexer rebalance exited due to panic - invalid memory address or nil pointer dereference-

{"stageInfo":{"index":{"totalProgress":20.06633028513428,"perNodeProgress":{"ns_1@172.23.104.233":0.2006633028513428,"ns_1@172.23.109.14":0.2006633028513428,"ns_1@172.23.106.115":0.2006633028513428,"ns_1@172.23.109.25":0.2006633028513428,"ns_1@172.23.109.15":0.2006633028513428},"startTime":"2023-12-01T12:27:44.101-08:00","completedTime":false,"timeTaken":22048},"data":{"totalProgress":100,"perNodeProgress":{"ns_1@172.23.106.192":1,"ns_1@172.23.106.119":1},"startTime":"2023-12-01T12:27:43.123-08:00","completedTime":"2023-12-01T12:27:44.058-08:00","timeTaken":934},"query":{"totalProgress":100,"perNodeProgress":{"ns_1@172.23.106.192":1,"ns_1@172.23.106.119":1},"startTime":"2023-12-01T12:27:44.058-08:00","completedTime":"2023-12-01T12:27:44.101-08:00","timeTaken":43}},"rebalanceId":"b04020fb335b674016bc54ff5020747d","nodesInfo":{"active_nodes":["ns_1@172.23.106.119","ns_1@172.23.106.192","ns_1@172.23.109.14","ns_1@172.23.109.15","ns_1@172.23.109.25","ns_1@172.23.104.233","ns_1@172.23.106.115"],"keep_nodes":["ns_1@172.23.106.119","ns_1@172.23.106.192","ns_1@172.23.109.14","ns_1@172.23.109.15","ns_1@172.23.109.25"],"eject_nodes":["ns_1@172.23.104.233","ns_1@172.23.106.115"],"delta_nodes":[],"failed_nodes":[]},"masterNode":"ns_1@172.23.106.115","startTime":"2023-12-01T12:27:43.117-08:00","completedTime":"2023-12-01T12:28:06.149-08:00","timeTaken":23032,"completionMessage":"Rebalance exited with reason {service_rebalance_failed,index,\n                              {agent_died,<27283.8457.0>,\n                               {lost_connection,\n                                {'ns_1@172.23.109.15',shutdown}}}}."}

2023-12-01T12:28:05.949-08:00 [Info] TK StreamBegin INIT_STREAM test_bucket:test_scope_1:test_collection_1 554 69079180830037 1104 9576 172.23.106.192:8091. HWT [1104-1104,1104].

2023-12-01T12:28:05.949-08:00 [Info] DATP[->dataport ":9103"] DATP -> Indexer 24.142544% blocked

2023-12-01T12:28:05.958-08:00 [Info] Shard /opt/couchbase/var/lib/couchbase/data/@2i/shards/shard9539116169749325165(9539116169749325165)(7910781805564410128-1-1) : evictPlasma: all instances:4 at resident 0

2023-12-01T12:28:05.958-08:00 [Info] controller '//opt/couchbase/var/lib/couchbase/data/@2i/shards/shard9539116169749325165': Controller.unregisterInstance: /opt/couchbase/var/lib/couchbase/data/@2i/test_bucket_hotelc62f2bf344ad47739bea1335cc972f0cflatten_keys_9361446216188480257_0.index/docIndex

2023-12-01T12:28:05.958-08:00 [Info] Shard /opt/couchbase/var/lib/couchbase/data/@2i/shards/shard13722083286406161335(13722083286406161335)(17692057394189471586-0-1) : Shard.doRecovery: Done recovery. Recovery log [/opt/couchbase/var/lib/couchbase/data/@2i/shards/shard13722083286406161335/data/recovery], Data log [/opt/couchbase/var/lib/couchbase/data/@2i/shards/shard13722083286406161335/data], Shared [true]. Built [2] plasmas, took [6.348762922s]

panic: runtime error: invalid memory address or nil pointer dereference

[signal SIGSEGV: segmentation violation code=0x1 addr=0x8 pc=0x1261620]

goroutine 3057731 [running]:

github.com/couchbase/plasma.(*Buffer).Grow(0x11ce002?, 0xc0112e5878?, 0x7f0b0fec7060?)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/util.go:116 +0x20

github.com/couchbase/plasma.(*Buffer).Get(...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/util.go:171

github.com/couchbase/plasma.(*page).marshalIndexKey(0x7f0b0cc8eae8?, 0x7f0b19d428e0, 0x2, 0x0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/page.go:1303 +0x45

github.com/couchbase/plasma.(*page).marshal(0xc018161540, 0x0, 0x0, 0x7f0affef43c0, 0x0?, 0x7f0b0f5a4b60, 0x0, 0x0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/page.go:1505 +0xdf

github.com/couchbase/plasma.(*page).compress(0xc018161540, 0x7f0b0f5a4b40)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/page.go:2419 +0x26b

github.com/couchbase/plasma.(*page).Compress(0xc018161540)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/page.go:2302 +0xe6

github.com/couchbase/plasma.(*Plasma).CompressPid(0xc00fea5500?, {0x3fe906f8ec9?}, 0x2ebc000?)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/swapper.go:527 +0x56

github.com/couchbase/plasma.(*Plasma).tryEvictPagesOnce(0xc00fea5500, 0xc01f257d60, 0x1, 0x0, 0xeb?)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/swapper.go:504 +0x405

github.com/couchbase/plasma.evictPlasma.func1(0xc0107daf00, {0xc009f9c730?, 0xc01b2c3300?, 0x2ebc040?})

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/recovery.go:2586 +0x2f9

github.com/couchbase/plasma.evictPlasma(0xc00622b040, 0xc004fe1980)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/recovery.go:2619 +0x13a

github.com/couchbase/plasma.(*Shard).recoverFromDataReplay.func3(0xc01b2cbd40?, {0x7f0ac4399d79, 0x3ba1, 0x8287}, 0x0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/recovery.go:1187 +0x2db

github.com/couchbase/plasma.(*lsStore).Visitor2(0xc014d76000?, 0x1a39213?, 0x2532c000, 0xc004e5db30, 0xc007cb8bc0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/lss.go:615 +0x10d

github.com/couchbase/plasma.(*LSSCtx).GetLSSVisitor(...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/lssctx.go:468

github.com/couchbase/plasma.(*Shard).recoverFromDataReplay(0xc004fe1980, 0xc00622b040, 0xc01a7de8c0, 0x22e2a000, 0xc01b315c20, 0xc0112e6698, 0xc00acf2d30)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/recovery.go:1203 +0x597

github.com/couchbase/plasma.(*Shard).doRecovery(0xc004fe1980, 0xc00622b040, 0xc01a7de8c0, 0xc01a7deb40, 0xc01b7e7500, 0x1, 0x0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/recovery.go:421 +0xfb3

github.com/couchbase/plasma.(*Shard).recoverFromShard.func2()

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/recovery.go:228 +0x3e

github.com/couchbase/plasma.(*Shard).recoverFromShard(0xc004fe1980, 0x0?, 0x0?, 0x0?, 0x0?, 0x1?, 0x0?)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/recovery.go:247 +0x105

github.com/couchbase/plasma.doRecovery({0x0?, 0x1e?}, 0x12c?, 0xc01b7e7500, 0xc004fe1980, 0xc01a7de8c0, 0x1b27470?, 0x1b274b8?)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/recovery.go:2154 +0xb7

github.com/couchbase/plasma.New6({_, _}, {_, _}, {{0x0, 0x0}, 0x1e, 0x12c, 0x5, 0x4, ...}, ...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/plasma.go:466 +0x4bf

github.com/couchbase/indexing/secondary/indexer.(*plasmaSlice).initStores.func3()

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:619 +0x39a

created by github.com/couchbase/indexing/secondary/indexer.(*plasmaSlice).initStores in goroutine 3057728

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:609 +0x32f

2023-12-01T12:28:06.742-08:00 [Info] Indexer started with command line: [/opt/couchbase/bin/indexer -adminPort=9100 -scanPort=9101 -httpPort=9102 -streamInitPort=9103 -streamCatchupPort=9104 -streamMaintPort=9105 --httpsPort=19102 --certFile=/opt/couchbase/var/lib/couchbase/config/certs/chain.pem --keyFile=/opt/couchbase/var/lib/couchbase/config/certs/pkey.pem --caFile=/opt/couchbase/var/lib/couchbase/config/certs/ca.pem --clientCertFile=/opt/couchbase/var/lib/couchbase/config/certs/client_chain.pem --clientKeyFile=/opt/couchbase/var/lib/couchbase/config/certs/client_pkey.pem -ipv4=required -ipv6=optional -cluster=127.0.0.1:8091 -storageDir=/opt/couchbase/var/lib/couchbase/data/@2i -diagDir=/opt/couchbase/var/lib/couchbase/crash -logDir=/opt/couchbase/var/lib/couchbase/logs