[System Test] [System Test upgrade] Indexer in panic loop - initializeBootstrapStats

2023-11-23T18:25:04.417-08:00 [Info] StorageMgr::updateIndexSnapMapForIndex, New IndexSnapshotContainer is being created for indexInst: 10601356723462398130

panic: runtime error: invalid memory address or nil pointer dereference

[signal SIGSEGV: segmentation violation code=0x1 addr=0x50 pc=0x142cafa]

goroutine 62 [running]:

github.com/couchbase/indexing/secondary/indexer.(*indexer).initializeBootstrapStats(0xc000480800, 0xc00592a700, 0x931f95c5872ce0b2)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:9067 +0x5a

github.com/couchbase/indexing/secondary/indexer.(*indexer).initFromPersistedState(0xc000480800)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:8991 +0xc0e

github.com/couchbase/indexing/secondary/indexer.(*indexer).bootstrap1.func1()

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:8299 +0x4bf

created by github.com/couchbase/indexing/secondary/indexer.(*indexer).bootstrap1 in goroutine 1

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:8268 +0x4c9

2023-11-23T18:25:09.734-08:00 [Info] Indexer started with command line: [/opt/couchbase/bin/indexer -adminPort=9100 -scanPort=9101 -httpPort=9102 -streamInitPort=9103 -streamCatchupPort=9104 -streamMaintPort=9105 --httpsPort=19102 --certFile=/opt/couchbase/var/lib/couchbase/config/certs/chain.pem --keyFile=/opt/couchbase/var/lib/couchbase/config/certs/pkey.pem --caFile=/opt/couchbase/var/lib/couchbase/config/certs/ca.pem --clientCertFile=/opt/couchbase/var/lib/couchbase/config/certs/client_chain.pem --clientKeyFile=/opt/couchbase/var/lib/couchbase/config/certs/client_pkey.pem -ipv4=required -ipv6=optional -cluster=127.0.0.1:8091 -storageDir=/data/@2i -diagDir=/opt/couchbase/var/lib/couchbase/crash -logDir=/opt/couchbase/var/lib/couchbase/logs -nodeUUID=3027c45f1a4421f55bd2d081faab0819 -isEnterprise=true --deploymentModel=default]

2023-11-23T18:25:09.734-08:00 [Info] Indexer DeploymentModel is set to: default

2023-11-23T18:25:09.734-08:00 [Info] Setting ipv6=false

2023-11-23T18:25:09.756-08:00 [Info] Indexer::NewIndexer Status Warmup

2023-11-23T18:25:09.772-08:00 [Info] IndexerSettingsManager: Setting numSliceWriters to 8 cgroup.max_cpu_percent: 800 runtime.NumCPU: 800

2023-11-23T18:25:09.777-08:00 [Info] Setting buffer block size to 16384 bytes

2023-11-23T18:25:09.777-08:00 [Info] settingsManager::setGlobalSettings: Indexer # CPU cores: 8, memory quota: 10096738304 bytes (9860096.000 KB, 9629.000 MB, 9.403 GB)

2023-11-23T18:25:09.777-08:00 [Info] Setting log level to Info

2023-11-23T18:25:09.777-08:00 [Warn] SetCICLMgrTimeDiffToForceFetch: Singleton Manager in ClusterInfoCacheLite is not set

2023-11-23T18:25:09.777-08:00 [Warn] SetCICLMgrSleepTimeOnNotifierRestart: Singleton Manager in ClusterInfoCacheLite is not set

2023-11-23T18:25:09.778-08:00 [Info] CpuThrottle::SetCpuTarget: New cpuTarget: 0.95

2023-11-23T18:25:09.778-08:00 [Info] audit::InitAuditService using address http://127.0.0.1:8091

2023-11-23T18:25:09.874-08:00 [Info] SettingsMgr::metaKVCallback Dropped request /indexing/settings/config {"indexer.plasma.backIndex.enableInMemoryCompression":true,"indexer.plasma.backIndex.enablePageBloomFilter":true,"indexer.plasma.mainIndex.enableInMemoryCompression":true,"indexer.plasma.recovery.enableFullReplayOnError":false,"indexer.settings.allow_large_keys":true,"indexer.settings.bufferPoolBlockSize":16384,"indexer.settings.build.batch_size":5,"indexer.settings.compaction.abort_exceed_interval":false,"indexer.settings.compaction.check_period":30,"indexer.settings.compaction.compaction_mode":"circular","indexer.settings.compaction.days_of_week":"Sunday,Monday,Tuesday,Wednesday,Thursday,Friday,Saturday","indexer.settings.compaction.interval":"00:00,00:00","indexer.settings.compaction.min_frag":30,"indexer.settings.compaction.min_size":524288000,"indexer.settings.compaction.plasma.manual":false,"indexer.settings.compaction.plasma.optional.decrement":5,"indexer.settings.compaction.plasma.optional.min_frag":20,"indexer.settings.compaction.plasma.optional.quota":25,"indexer.settings.corrupt_index_num_backups":1,"indexer.settings.cpuProfDir":"","indexer.settings.cpuProfile":false,"indexer.settings.eTagPeriod":240,"indexer.settings.enable_corrupt_index_backup":true,"indexer.settings.enable_page_bloom_filter":true,"indexer.settings.enable_shard_affinity":true,"indexer.settings.fast_flush_mode":true,"indexer.settings.gc_percent":100,"indexer.settings.inmemory_snapshot.fdb.interval":200,"indexer.settings.inmemory_snapshot.interval":200,"indexer.settings.inmemory_snapshot.moi.interval":10,"indexer.settings.largeSnapshotThreshold":200,"indexer.settings.log_level":"info","indexer.settings.maxNumPartitions":64,"indexer.settings.maxVbQueueLength":0,"indexer.settings.max_array_seckey_size":10240,"indexer.settings.max_cpu_percent":0,"indexer.settings.max_seckey_size":4608,"indexer.settings.max_writer_lock_prob":20,"indexer.settings.memProfDir":"","indexer.settings.memProfile":false,"indexer.settings.memory_quota":10096738304,"indexer.settings.minVbQueueLength":30,"indexer.settings.moi.debug":false,"indexer.settings.moi.persistence_threads":4,"indexer.settings.moi.recovery.max_rollbacks":2,"indexer.settings.moi.recovery_threads":8,"indexer.settings.num_replica":1,"indexer.settings.percentage_memory_quota":0,"indexer.settings.persisted_snapshot.fdb.interval":5000,"indexer.settings.persisted_snapshot.interval":5000,"indexer.settings.persisted_snapshot.moi.interval":600000,"indexer.settings.persisted_snapshot_init_build.fdb.interval":5000,"indexer.settings.persisted_snapshot_init_build.interval":5000,"indexer.settings.persisted_snapshot_init_build.moi.interval":600000,"indexer.settings.plasma.recovery.max_rollbacks":2,"indexer.settings.rebalance.blob_storage_bucket":"","indexer.settings.rebalance.blob_storage_prefix":"","indexer.settings.rebalance.blob_storage_region":"","indexer.settings.rebalance.blob_storage_scheme":"","indexer.settings.rebalance.redistribute_indexes":true,"indexer.settings.recovery.max_rollbacks":2,"indexer.settings.scan_getseqnos_retries":30,"indexer.settings.scan_timeout":120000,"indexer.settings.send_buffer_size":1024,"indexer.settings.serverless.indexLimit":201,"indexer.settings.sliceBufSize":1600,"indexer.settings.smallSnapshotThreshold":30,"indexer.settings.snapshotListeners":4,"indexer.settings.snapshotRequestWorkers":4,"indexer.settings.statsLogDumpInterval":60,"indexer.settings.storage_mode":"plasma","indexer.settings.storage_mode.disable_upgrade":false,"indexer.settings.thresholds.mem_high":70,"indexer.settings.thresholds.mem_low":50,"indexer.settings.thresholds.units_high":60,"indexer.settings.thresholds.units_low":40,"indexer.settings.units_quota":10000,"indexer.settings.wal_size":4096,"projector.settings.log_level":"info","queryport.client.settings.backfillLimit":5120,"queryport.client.settings.closeActiveConnections":true,"queryport.client.settings.minPoolSizeWM":1000,"queryport.client.settings.poolOverflow":30,"queryport.client.settings.poolSize":5000,"queryport.client.settings.relConnBatchSize":100}. Any setting change will get applied once Indexer is ready.

2023/11/23 18:25:09 audit: created new audit service

2023-11-23T18:25:09.952-08:00 [Info] InitSystemEventLogger: Started SystemEventLogger

2023-11-23T18:25:09.952-08:00 [Info] Receive security change notification. code 7

2023-11-23T18:25:09.952-08:00 [Info] TLS config refreshed successfully

2023-11-23T18:25:09.953-08:00 [Info] Certificate refreshed successfully with certFile /opt/couchbase/var/lib/couchbase/config/certs/chain.pem, keyFile /opt/couchbase/var/lib/couchbase/config/certs/pkey.pem, caFile /opt/couchbase/var/lib/couchbase/config/certs/ca.pem

2023-11-23T18:25:09.953-08:00 [Info] Encryption config refresh successfully.   Encryption enabled=false

2023-11-23T18:25:09.953-08:00 [Info] security context:  encryptLocalHost true Local IP's: map[127.0.0.1:true 172.23.97.109:true ::1:true fd63:6f75:6368:2060:24e5:7f68:638c:ab0e:true fd63:6f75:6368:2060:51bd:2a66:445f:38e4:true fd63:6f75:6368:2060:6101:6e2d:8edf:d953:true fd63:6f75:6368:2060:8498:6eff:fea4:1843:true fd63:6f75:6368:2060:a8b4:f9:a8d7:c5bd:true fd63:6f75:6368:2060:bcf3:a9e2:dce6:9ce7:true fd63:6f75:6368:2060:dd7c:787f:1b36:c523:true fd63:6f75:6368:2060:e42a:66cc:a54c:33a:true fe80::8498:6eff:fea4:1843:true]

2023-11-23T18:25:10.034-08:00 [Info] security context initialized

2023-11-23T18:25:10.035-08:00 [Info] security port mapping updated : map[11210:11207 8091:18091 8093:18093 9100:9100 9101:9101 9102:19102 9103:9103 9104:9104 9105:9105 9999:9999]

2023-11-23T18:25:10.035-08:00 [Info] Indexer::NewIndexer Build Mode Set Enterprise

2023-11-23T18:25:10.035-08:00 [Info] Indexer::Cluster Storage Mode Set plasma

2023-11-23T18:25:10.035-08:00 [Info] memcachedTimeout set to 120

2023-11-23T18:25:10.035-08:00 [Info] Indexer::NewIndexer done initializing from config

2023-11-23T18:25:10.038-08:00 [Info] SetMajorVersionCICL: Setting cluster version to 7

2023-11-23T18:25:10.038-08:00 [Info] newClusterInfoCacheLiteManager: started New clusterInfoCacheManager

2023-11-23T18:25:10.038-08:00 [Info] NewClusterInfoCacheLiteClient started new cicl client for indexer

2023-11-23T18:25:10.039-08:00 [Warn] PollForDeletedBucketsV2: Falling back to pollForDeletedBuckets() as use_cinfo_lite is false or not present

2023-11-23T18:25:10.042-08:00 [Info] mutationMgr::setEnableAuth: enableAuth set to 1

2023-11-23T18:25:10.047-08:00 [Info] MutationMgr::checkPortAvailability for dataport servers ...

2023-11-23T18:25:10.047-08:00 [Info] MutationMgr::checkPortAvailability all dataport server ports are available.

2023-11-23T18:25:10.048-08:00 [Info] serviceChangeNotifier: Creating new notifier instance for http://%40index@127.0.0.1:8091-default

2023-11-23T18:25:10.051-08:00 [Info] serviceChangeNotifier: received ServiceChangeNotification

2023-11-23T18:25:10.071-08:00 [Info] Terminating WatchClusterVersionChanges: Cluster version is >= 9

2023-11-23T18:25:10.071-08:00 [Info] serviceChangeNotifier: Starting to monitor the bucket streaming endpoint for bucket: bucket1

2023-11-23T18:25:10.071-08:00 [Info] serviceChangeNotifier: Starting to monitor the bucket streaming endpoint for bucket: bucket2

2023-11-23T18:25:10.071-08:00 [Info] serviceChangeNotifier: Starting to monitor the bucket streaming endpoint for bucket: bucket3

2023-11-23T18:25:10.071-08:00 [Info] serviceChangeNotifier: Starting to monitor the bucket streaming endpoint for bucket: bucket4

2023-11-23T18:25:10.071-08:00 [Info] serviceChangeNotifier: Starting to monitor the bucket streaming endpoint for bucket: bucket5

2023-11-23T18:25:10.071-08:00 [Info] serviceChangeNotifier: received PoolChangeNotification

2023-11-23T18:25:10.128-08:00 [Info] Timekeeper::setMaxTsQueueLen 2000

2023-11-23T18:25:10.128-08:00 [Info] OperationsMonitor(kvSender) starting monitor ...

2023-11-23T18:25:10.128-08:00 [Info] [Queryport ":9101"] started ...

2023-11-23T18:25:10.131-08:00 [Info] Unable to fetch storage mode from metakv during bootrap.  Use storage mode setting for bootstrap

2023-11-23T18:25:10.133-08:00 [Info] Indexer::NewIndexer bootstrap storage mode plasma

2023-11-23T18:25:10.133-08:00 [Info] NewClusterInfoCacheLiteClient started new cicl client for IndexMgr

2023-11-23T18:25:10.134-08:00 [Info] NewClusterInfoCacheLiteClient started new cicl client for RequestHandler

2023-11-23T18:25:10.134-08:00 [Info] NewClusterInfoCacheLiteClient started new cicl client for LifecycleMgr

2023-11-23T18:25:10.135-08:00 [Info] Starting metadadta repo: quota 1048576 sleep duration 900 threshold 30 min file size 0

2023-11-23T18:25:10.135-08:00 [INFO][FDB] Forestdb blockcache size 1048576 initialized in 166 us

2023-11-23T18:25:10.136-08:00 [INFO][FDB] Forestdb opened database file /data/@2i/MetadataStore.2

2023-11-23T18:25:10.138-08:00 [Info] Repo.upgradeAndOpenDBFile(/data/@2i/MetadataStore): Opened with COMPACT_AUTO mode

2023-11-23T18:25:10.138-08:00 [Info] EmbeddedServer.runOnce() : Start Running Server

2023-11-23T18:25:10.148-08:00 [Info] indexer:: Staring http server : :9102

2023-11-23T18:25:10.148-08:00 [Info] HTTPS server created for :19102

2023-11-23T18:25:10.148-08:00 [Info] indexer:: SSL server started: :19102

2023-11-23T18:25:10.148-08:00 [Info] Indexer::indexer version 9 with priority 7.6.0

2023-11-23T18:25:10.148-08:00 [Info] ClustMgr:handleGetLocalValue Key IndexerId

2023-11-23T18:25:10.148-08:00 [Info] Indexer Id 3027c45f1a4421f55bd2d081faab0819

2023-11-23T18:25:10.148-08:00 [Info] ClustMgr:handleGetLocalValue Key RebalanceRunning

2023-11-23T18:25:10.148-08:00 [Info] ClustMgr:handleGetLocalValue Key RebalanceToken

2023-11-23T18:25:10.148-08:00 [Info] Indexer::recoverRebalanceState RebalanceRunning true RebalanceToken &{3027c45f1a4421f55bd2d081faab0819 9f43e41008041babd16c4f9ee48f945d Rebalance  172.23.97.109 0 REBALANCE_NOT_RUNNING SHARD}

2023-11-23T18:25:10.148-08:00 [Info] ClustMgr:handleGetLocalValuesWithPrefix Key PauseResumeRunning

2023-11-23T18:25:10.148-08:00 [Info] ClustMgr:handleGetLocalValuesWithPrefix Key PauseToken

2023-11-23T18:25:10.149-08:00 [Info] ClustMgr:handleGetGlobalTopology indexInstMap : 

2023-11-23T18:25:10.190-08:00 [Info] Indexer::initFromPersistedState Recovered IndexInstMap 

	InstanceId: 10601356723462398130 Name: idx8_VAOS9H7C47_idxprefix Keyspace: bucket5:_default:_default State: INDEX_STATE_ACTIVE Stream: MAINT_STREAM RState: RebalPending Version: 1 ReplicaId: 2 

	InstanceId: 9515757715472060864 Name: idx8_VAOS9H7C47_idxprefix Keyspace: bucket5:_default:_default State: INDEX_STATE_ACTIVE Stream: MAINT_STREAM RState: RebalPending Version: 1 ReplicaId: 2 RealInstId: 10601356723462398130 

, elapsed: 41.386146ms

2023-11-23T18:25:10.190-08:00 [Info] DDLServiceMgr: intialized. Local nodeUUID 3027c45f1a4421f55bd2d081faab0819

2023-11-23T18:25:10.190-08:00 [Info] NewClusterInfoCacheLiteClient started new cicl client for schedIndexCreator

2023-11-23T18:25:10.190-08:00 [Info] schedIndexCreator: intialized.

2023-11-23T18:25:10.190-08:00 [Info] RebalanceServiceManager::NewRebalanceServiceManager true &{3027c45f1a4421f55bd2d081faab0819 9f43e41008041babd16c4f9ee48f945d Rebalance  172.23.97.109 0 REBALANCE_NOT_RUNNING SHARD} 

2023-11-23T18:25:10.190-08:00 [Info] DDLServiceMgr::runTokenCleaner: Starting with period 5m0s

2023-11-23T18:25:10.190-08:00 [Info] RebalanceServiceManager::initService Init

2023-11-23T18:25:10.190-08:00 [Info] MasterServiceManager::registerWithServer: *indexer.MasterServiceManager implements service.AutofailoverManager; *indexer.MasterServiceManager implements service.Manager; *indexer.MasterServiceManager implements service.ServerlessManager

2023-11-23T18:25:10.192-08:00 [Info] GenericServiceManager::GetTaskList: called with rev: []

2023-11-23T18:25:10.192-08:00 [Info] GenericServiceManager::GetTaskList: return from rev [] call. taskList: &{Rev:[0 0 0 0 0 0 0 0] Tasks:[]}

2023-11-23T18:25:10.194-08:00 [Info] RebalanceServiceManager::GetCurrentTopology []

2023-11-23T18:25:10.194-08:00 [Info] RebalanceServiceManager::GetCurrentTopology returns &{Rev:[0 0 0 0 0 0 0 0] Nodes:[3027c45f1a4421f55bd2d081faab0819] IsBalanced:false Messages:[]}

2023-11-23T18:25:10.194-08:00 [Info] GenericServiceManager::GetTaskList: called with rev: [0 0 0 0 0 0 0 0]

2023-11-23T18:25:10.194-08:00 [Info] RebalanceServiceManager::GetCurrentTopology [0 0 0 0 0 0 0 0]

2023-11-23T18:25:10.222-08:00 [Info] KVSender::sendShutdownTopic Projector 172.23.96.198:9999 Topic MAINT_STREAM_TOPIC_3027c45f1a4421f55bd2d081faab0819

2023-11-23T18:25:10.225-08:00 [Error] KVSender::sendShutdownTopic Unexpected Error During Shutdown Projector 172.23.96.198:9999 Topic MAINT_STREAM_TOPIC_3027c45f1a4421f55bd2d081faab0819. Err projector.topicMissing

2023-11-23T18:25:10.225-08:00 [Error] KVSender::closeMutationStream MAINT_STREAM  Error Received projector.topicMissing from 172.23.96.198:9999

2023-11-23T18:25:10.225-08:00 [Info] KVSender::closeMutationStream MAINT_STREAM  Treating projector.topicMissing As Success

2023-11-23T18:25:10.226-08:00 [Info] KVSender::sendShutdownTopic Projector 172.23.96.230:9999 Topic MAINT_STREAM_TOPIC_3027c45f1a4421f55bd2d081faab0819

2023-11-23T18:25:10.227-08:00 [Error] KVSender::sendShutdownTopic Unexpected Error During Shutdown Projector 172.23.96.230:9999 Topic MAINT_STREAM_TOPIC_3027c45f1a4421f55bd2d081faab0819. Err projector.topicMissing

2023-11-23T18:25:10.227-08:00 [Error] KVSender::closeMutationStream MAINT_STREAM  Error Received projector.topicMissing from 172.23.96.230:9999

2023-11-23T18:25:10.227-08:00 [Info] KVSender::closeMutationStream MAINT_STREAM  Treating projector.topicMissing As Success

2023-11-23T18:25:10.227-08:00 [Info] KVSender::sendShutdownTopic Projector 172.23.96.245:9999 Topic MAINT_STREAM_TOPIC_3027c45f1a4421f55bd2d081faab0819

2023-11-23T18:25:10.229-08:00 [Error] KVSender::sendShutdownTopic Unexpected Error During Shutdown Projector 172.23.96.245:9999 Topic MAINT_STREAM_TOPIC_3027c45f1a4421f55bd2d081faab0819. Err projector.topicMissing

2023-11-23T18:25:10.229-08:00 [Error] KVSender::closeMutationStream MAINT_STREAM  Error Received projector.topicMissing from 172.23.96.245:9999

2023-11-23T18:25:10.229-08:00 [Info] KVSender::closeMutationStream MAINT_STREAM  Treating projector.topicMissing As Success

2023-11-23T18:25:10.230-08:00 [Info] KVSender::sendShutdownTopic Projector 172.23.97.66:9999 Topic MAINT_STREAM_TOPIC_3027c45f1a4421f55bd2d081faab0819

2023-11-23T18:25:10.232-08:00 [Error] KVSender::sendShutdownTopic Unexpected Error During Shutdown Projector 172.23.97.66:9999 Topic MAINT_STREAM_TOPIC_3027c45f1a4421f55bd2d081faab0819. Err projector.topicMissing

2023-11-23T18:25:10.232-08:00 [Error] KVSender::closeMutationStream MAINT_STREAM  Error Received projector.topicMissing from 172.23.97.66:9999

2023-11-23T18:25:10.232-08:00 [Info] KVSender::closeMutationStream MAINT_STREAM  Treating projector.topicMissing As Success

2023-11-23T18:25:10.232-08:00 [Info] KVSender::sendShutdownTopic Projector 172.23.96.198:9999 Topic INIT_STREAM_TOPIC_3027c45f1a4421f55bd2d081faab0819

2023-11-23T18:25:10.233-08:00 [Error] KVSender::sendShutdownTopic Unexpected Error During Shutdown Projector 172.23.96.198:9999 Topic INIT_STREAM_TOPIC_3027c45f1a4421f55bd2d081faab0819. Err projector.topicMissing

2023-11-23T18:25:10.233-08:00 [Error] KVSender::closeMutationStream INIT_STREAM  Error Received projector.topicMissing from 172.23.96.198:9999

2023-11-23T18:25:10.233-08:00 [Info] KVSender::closeMutationStream INIT_STREAM  Treating projector.topicMissing As Success

2023-11-23T18:25:10.233-08:00 [Info] KVSender::sendShutdownTopic Projector 172.23.96.230:9999 Topic INIT_STREAM_TOPIC_3027c45f1a4421f55bd2d081faab0819

2023-11-23T18:25:10.234-08:00 [Error] KVSender::sendShutdownTopic Unexpected Error During Shutdown Projector 172.23.96.230:9999 Topic INIT_STREAM_TOPIC_3027c45f1a4421f55bd2d081faab0819. Err projector.topicMissing

2023-11-23T18:25:10.234-08:00 [Error] KVSender::closeMutationStream INIT_STREAM  Error Received projector.topicMissing from 172.23.96.230:9999

2023-11-23T18:25:10.234-08:00 [Info] KVSender::closeMutationStream INIT_STREAM  Treating projector.topicMissing As Success

2023-11-23T18:25:10.234-08:00 [Info] KVSender::sendShutdownTopic Projector 172.23.96.245:9999 Topic INIT_STREAM_TOPIC_3027c45f1a4421f55bd2d081faab0819

2023-11-23T18:25:10.234-08:00 [Error] KVSender::sendShutdownTopic Unexpected Error During Shutdown Projector 172.23.96.245:9999 Topic INIT_STREAM_TOPIC_3027c45f1a4421f55bd2d081faab0819. Err projector.topicMissing

2023-11-23T18:25:10.234-08:00 [Error] KVSender::closeMutationStream INIT_STREAM  Error Received projector.topicMissing from 172.23.96.245:9999

2023-11-23T18:25:10.234-08:00 [Info] KVSender::closeMutationStream INIT_STREAM  Treating projector.topicMissing As Success

2023-11-23T18:25:10.234-08:00 [Info] internalVersionMonitor:monitor starting. Term versions (9:7.6.0)

2023-11-23T18:25:10.234-08:00 [Info] KVSender::sendShutdownTopic Projector 172.23.97.66:9999 Topic INIT_STREAM_TOPIC_3027c45f1a4421f55bd2d081faab0819

2023-11-23T18:25:10.234-08:00 [Info] internalVersionMonitor:ticker starting ...

2023-11-23T18:25:10.235-08:00 [Error] KVSender::sendShutdownTopic Unexpected Error During Shutdown Projector 172.23.97.66:9999 Topic INIT_STREAM_TOPIC_3027c45f1a4421f55bd2d081faab0819. Err projector.topicMissing

2023-11-23T18:25:10.235-08:00 [Error] KVSender::closeMutationStream INIT_STREAM  Error Received projector.topicMissing from 172.23.97.66:9999

2023-11-23T18:25:10.235-08:00 [Info] KVSender::closeMutationStream INIT_STREAM  Treating projector.topicMissing As Success

2023-11-23T18:25:10.235-08:00 [Info] ClustMgr:handleUpdateTopologyForIndex mType: CLUST_MGR_UPDATE_TOPOLOGY_FOR_INDEX indexList: [] bucket:  scope:  collection:  streamId: NIL_STREAM syncUpdate: false respCh: <nil> 

2023-11-23T18:25:10.237-08:00 [Info] internalVersionMonitor:notifier starting ...

2023-11-23T18:25:10.237-08:00 [Info] internalVersionMonitor:monitor terminate. Cluster version reached 9

2023-11-23T18:25:10.237-08:00 [Info] internalVersionMonitor:ticker stopping ...

2023-11-23T18:25:10.240-08:00 [Info] indexer.upgradeStorage: check index for storage upgrade.   disable false overrride not_set

2023-11-23T18:25:10.240-08:00 [Info] Indexer::local storage mode plasma

2023-11-23T18:25:10.240-08:00 [Info] StorageMgr::updateIndexSnapMapForIndex, New IndexSnapshotContainer is being created for indexInst: 10601356723462398130

panic: runtime error: invalid memory address or nil pointer dereference

[signal SIGSEGV: segmentation violation code=0x1 addr=0x50 pc=0x142cafa]

goroutine 248 [running]:

github.com/couchbase/indexing/secondary/indexer.(*indexer).initializeBootstrapStats(0xc000100400, 0xc00596e380, 0x931f95c5872ce0b2)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:9067 +0x5a

github.com/couchbase/indexing/secondary/indexer.(*indexer).initFromPersistedState(0xc000100400)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:8991 +0xc0e

github.com/couchbase/indexing/secondary/indexer.(*indexer).bootstrap1.func1()

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:8299 +0x4bf

created by github.com/couchbase/indexing/secondary/indexer.(*indexer).bootstrap1 in goroutine 1

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:8268 +0x4c9

2023-11-23T18:25:15.615-08:00 [Info] Indexer started with command line: [/opt/couchbase/bin/indexer -adminPort=9100 -scanPort=9101 -httpPort=9102 -streamInitPort=9103 -streamCatchupPort=9104 -streamMaintPort=9105 --httpsPort=19102 --certFile=/opt/couchbase/var/lib/couchbase/config/certs/chain.pem --keyFile=/opt/couchbase/var/lib/couchbase/config/certs/pkey.pem --caFile=/opt/couchbase/var/lib/couchbase/config/certs/ca.pem --clientCertFile=/opt/couchbase/var/lib/couchbase/config/certs/client_chain.pem --clientKeyFile=/opt/couchbase/var/lib/couchbase/config/certs/client_pkey.pem -ipv4=required -ipv6=optional -cluster=127.0.0.1:8091 -storageDir=/data/@2i -diagDir=/opt/couchbase/var/lib/couchbase/crash -logDir=/opt/couchbase/var/lib/couchbase/logs -nodeUUID=3027c45f1a4421f55bd2d081faab0819 -isEnterprise=true --deploymentModel=default]