[BP to 7.1.4] [System Test] Panic seen in indexer

fatal error: concurrent map read and map write

goroutine 29577 [running]:

runtime.throw({0x181dfc0?, 0xc016affbc0?})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/runtime/panic.go:992 +0x71 fp=0xc0059246b0 sp=0xc005924680 pc=0x43f7d1

runtime.mapaccess1_faststr(0x159a220?, 0xc0000c4a80?, {0xc002e160f0, 0x7})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/runtime/map_faststr.go:22 +0x3a5 fp=0xc005924718 sp=0xc0059246b0 pc=0x41a0a5

github.com/couchbase/indexing/secondary/indexer.(*indexer).ValidateKeyspace(0xc000124000, 0x4ae0?, {0xc002e160f0, 0x7}, {0xc02a7bf200, 0xb, 0x10})

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:11172 +0xa8 fp=0xc0059247d8 sp=0xc005924718 pc=0x129fe88

github.com/couchbase/indexing/secondary/indexer.(*indexer).startKeyspaceIdStream.func1(0xc02ab52840?)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:7335 +0x1fa fp=0xc005924fc8 sp=0xc0059247d8 pc=0x128593a

github.com/couchbase/indexing/secondary/indexer.(*indexer).startKeyspaceIdStream.func2()

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:7510 +0x2a fp=0xc005924fe0 sp=0xc005924fc8 pc=0x128570a

runtime.goexit()

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/runtime/asm_amd64.s:1571 +0x1 fp=0xc005924fe8 sp=0xc005924fe0 pc=0x472501

created by github.com/couchbase/indexing/secondary/indexer.(*indexer).startKeyspaceIdStream

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:7328 +0x1b7b

goroutine 1 [runnable]:

github.com/couchbase/indexing/secondary/indexer.(*indexer).addToInstsPerCollMap(0xc000124000, {0xc02b0a8000, 0xb, 0x8?})

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:11389 +0x210

github.com/couchbase/indexing/secondary/indexer.(*indexer).startKeyspaceIdStream(0xc000124000, 0x1, {0xc002dd8c90, 0x8}, 0xc01a6da000, 0x0, {0x0, 0x0, 0x0}, 0xc01d4cf500, ...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:7274 +0x1034

github.com/couchbase/indexing/secondary/indexer.(*indexer).startStreams(0xc000124000)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:9073 +0x165

github.com/couchbase/indexing/secondary/indexer.(*indexer).bootstrap2(0xc000124000)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:7994 +0x69a

github.com/couchbase/indexing/secondary/indexer.(*indexer).handleStorageWarmupDone(0xc000124000, {0x1c192e0?, 0xc00feb8cd8?})

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:7867 +0x449

github.com/couchbase/indexing/secondary/indexer.(*indexer).handleWorkerMsgs(0xc000124000, {0x1c192e0, 0xc00feb8cd8})

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:1519 +0x128f

github.com/couchbase/indexing/secondary/indexer.(*indexer).run(0xc000124000)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:1128 +0x36e

github.com/couchbase/indexing/secondary/indexer.NewIndexer(0x2b6c710?)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:554 +0x2989

main.main()

	goproj/src/github.com/couchbase/indexing/secondary/cmd/indexer/main.go:206 +0x11d3

goroutine 21 [select]:

github.com/couchbase/cbauth/cbauthimpl.(*tlsNotifier).loop(0xc00012c6a8)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/cbauthimpl/impl.go:405 +0x67

created by github.com/couchbase/cbauth/cbauthimpl.NewSVCForTest

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/cbauthimpl/impl.go:565 +0x37a

goroutine 22 [select]:

github.com/couchbase/cbauth/cbauthimpl.(*cfgChangeNotifier).loop(0xc00012c6c0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/cbauthimpl/impl.go:325 +0x85

created by github.com/couchbase/cbauth/cbauthimpl.NewSVCForTest

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/cbauthimpl/impl.go:566 +0x3ca

goroutine 23 [IO wait]:

internal/poll.runtime_pollWait(0x7fd36d9b2fc8, 0x72)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/runtime/netpoll.go:302 +0x89

internal/poll.(*pollDesc).wait(0xc0000ac100?, 0xc0000bd000?, 0x0)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_poll_runtime.go:83 +0x32

internal/poll.(*pollDesc).waitRead(...)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_poll_runtime.go:88

internal/poll.(*FD).Read(0xc0000ac100, {0xc0000bd000, 0x1000, 0x1000})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_unix.go:167 +0x25a

net.(*netFD).Read(0xc0000ac100, {0xc0000bd000?, 0xc00038a010?, 0xc00013f958?})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/fd_posix.go:55 +0x29

net.(*conn).Read(0xc000010020, {0xc0000bd000?, 0x11?, 0xc00014d9e8?})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/net.go:183 +0x45

bufio.(*Reader).Read(0xc0000a8120, {0xc000382001, 0xdff, 0x456ef4?})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/bufio/bufio.go:236 +0x1b4

github.com/couchbase/cbauth/revrpc.(*minirwc).Read(0x191?, {0xc000382001?, 0x8?, 0x20?})

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/revrpc/revrpc.go:103 +0x25

encoding/json.(*Decoder).refill(0xc0000ca000)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/encoding/json/stream.go:165 +0x17f

encoding/json.(*Decoder).readValue(0xc0000ca000)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/encoding/json/stream.go:140 +0xbb

encoding/json.(*Decoder).Decode(0xc0000ca000, {0x158a3a0, 0xc0000a8260})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/encoding/json/stream.go:63 +0x78

net/rpc/jsonrpc.(*serverCodec).ReadRequestHeader(0xc0000a8240, 0xc00007c0a0)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/rpc/jsonrpc/server.go:66 +0x85

net/rpc.(*Server).readRequestHeader(0xc0000c6000, {0x1c274c0, 0xc00007a1d0})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/rpc/server.go:587 +0x66

net/rpc.(*Server).readRequest(0x0?, {0x1c274c0, 0xc00007a1d0})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/rpc/server.go:547 +0x3b

net/rpc.(*Server).ServeCodec(0xc0000c6000, {0x1c274c0?, 0xc00007a1d0})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/rpc/server.go:462 +0x87

github.com/couchbase/cbauth/revrpc.(*Service).Run(0xc000071f60?, 0xc00008dfa0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/revrpc/revrpc.go:236 +0x77c

github.com/couchbase/cbauth/revrpc.BabysitService(0x0?, 0x0?, {0x1c1ae00?, 0xc00000e060?})

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/revrpc/revrpc.go:335 +0x58

github.com/couchbase/cbauth.runRPCForSvc(0x0?, 0xc0001901c0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/default.go:57 +0xbd

github.com/couchbase/cbauth.startDefault.func1()

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/default.go:66 +0x25

created by github.com/couchbase/cbauth.startDefault

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/default.go:65 +0xf9

goroutine 240 [select]:

github.com/couchbase/indexing/secondary/dcp.(*Client).runObserveStreamingEndpoint(0xc000003520?, {0xc000556528, 0x17}, 0xc000542798, 0xc000542780, 0xc003e98600)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/dcp/pools.go:503 +0x6da

github.com/couchbase/indexing/secondary/dcp.(*Client).RunObservePool(0xc003ea4000?, {0x17ec83f?, 0xc002b7b920?}, 0xc002c58000?, 0x163d840?)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/dcp/pools.go:395 +0xb2

github.com/couchbase/indexing/secondary/common.(*serviceNotifierInstance).RunPoolObserver(0xc003ea4000)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/services_notifier.go:147 +0x5c

created by github.com/couchbase/indexing/secondary/common.NewServicesChangeNotifier

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/services_notifier.go:256 +0x50a

goroutine 9 [chan receive]:

github.com/couchbase/plasma.(*smrManager).run(0xc000669800)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/smr.go:526 +0x96

created by github.com/couchbase/plasma.NewSmrManager

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/smr.go:435 +0xaf

goroutine 7 [select]:

net/http.(*persistConn).readLoop(0xc0000b2360)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:2213 +0xd99

created by net/http.(*Transport).dialConn

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:1750 +0x173e

goroutine 8 [select]:

net/http.(*persistConn).writeLoop(0xc0000b2360)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:2392 +0xf5

created by net/http.(*Transport).dialConn

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:1751 +0x1791

goroutine 24 [chan receive]:

github.com/couchbase/regulator/utils/logging.duplicateFlushLoop()

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/regulator/utils/logging/logger_golog.go:253 +0x25

created by github.com/couchbase/regulator/utils/logging.init.0

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/regulator/utils/logging/logger_golog.go:49 +0x7e

goroutine 10 [chan receive]:

github.com/couchbase/plasma.runCleanerAutoTuner()

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/auto_tuner.go:653 +0x194

created by github.com/couchbase/plasma.init.2

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/shard.go:211 +0x1fa

goroutine 11 [chan receive]:

github.com/couchbase/plasma.singletonWorker()

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/shard.go:4871 +0xb0

created by github.com/couchbase/plasma.init.2

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/shard.go:212 +0x206

goroutine 12 [chan receive]:

github.com/couchbase/plasma.systemResourceTracker()

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/mem.go:541 +0xa5

created by github.com/couchbase/plasma.init.2

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/shard.go:213 +0x212

goroutine 13 [chan receive]:

github.com/couchbase/plasma.AggregateAndLogStats()

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/shard.go:4832 +0x147

created by github.com/couchbase/plasma.init.2

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/shard.go:214 +0x21e

goroutine 32 [chan receive]:

github.com/couchbase/plasma.(*CleanerAutoTuner).refreshCleanerBandwidth(0xc0003d8100)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/auto_tuner.go:664 +0x86

created by github.com/couchbase/plasma.runCleanerAutoTuner

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/auto_tuner.go:650 +0x136

goroutine 14 [chan receive]:

github.com/couchbase/plasma.(*TenantMgr).Run(0xc00020b360)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/tenant.go:419 +0x9d

created by github.com/couchbase/plasma.init.3

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/tenant.go:57 +0x5d

goroutine 51 [syscall]:

syscall.Syscall(0x0, 0x0, 0xc0003c27ac, 0x4)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/syscall/asm_linux_amd64.s:20 +0x5

syscall.read(0xc000112060?, {0xc0003c27ac?, 0x3?, 0x10000c0000021a0?})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/syscall/zsyscall_linux_amd64.go:696 +0x4d

syscall.Read(...)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/syscall/syscall_unix.go:188

internal/poll.ignoringEINTRIO(...)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_unix.go:794

internal/poll.(*FD).Read(0xc000112060?, {0xc0003c27ac?, 0x4?, 0x4?})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_unix.go:163 +0x285

os.(*File).read(...)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/os/file_posix.go:31

os.(*File).Read(0xc000134000, {0xc0003c27ac?, 0xc0001fa930?, 0x18be388?})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/os/file.go:119 +0x5e

github.com/couchbase/indexing/secondary/common.ExitOnStdinClose()

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/util.go:440 +0x39

created by main.main

	goproj/src/github.com/couchbase/indexing/secondary/cmd/indexer/main.go:110 +0xb9b

goroutine 16 [select]:

github.com/couchbase/cbauth/metakv.doRunObserveChildren(0xc0001c2420?, {0x1c14610, 0x1}, 0xc00013fe88, 0xc00022e7e0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:301 +0x429

github.com/couchbase/cbauth/metakv.(*store).runObserveChildren(...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:259

github.com/couchbase/cbauth/metakv.RunObserveChildren({0x1c14610?, 0x0?}, 0x0?, 0x0?)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:389 +0x58

github.com/couchbase/indexing/secondary/indexer.NewSettingsManager.func1.1(0x0?, {0x0?, 0x0?})

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/settings.go:103 +0xcc

github.com/couchbase/indexing/secondary/common.(*RetryHelper).Run(0xc00013ffa8)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/retry_helper.go:36 +0x83

github.com/couchbase/indexing/secondary/indexer.NewSettingsManager.func1()

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/settings.go:107 +0x75

created by github.com/couchbase/indexing/secondary/indexer.NewSettingsManager

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/settings.go:98 +0x54a

goroutine 66 [chan receive]:

github.com/couchbase/indexing/secondary/indexer.(*settingsManager).run(0xc000486540)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/settings.go:315 +0x4e

created by github.com/couchbase/indexing/secondary/indexer.NewSettingsManager

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/settings.go:114 +0x58d

goroutine 71 [chan receive]:

github.com/couchbase/indexing/secondary/common.(*clusterInfoCacheLiteManager).handlePoolsChangeNotifications(0xc002bba3c0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/cluster_info_lite.go:1079 +0x66

created by github.com/couchbase/indexing/secondary/common.newClusterInfoCacheLiteManager

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/cluster_info_lite.go:808 +0x893

goroutine 33 [IO wait]:

internal/poll.runtime_pollWait(0x7fd36d9b2ed8, 0x72)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/runtime/netpoll.go:302 +0x89

internal/poll.(*pollDesc).wait(0xc0000ac480?, 0xc0000de000?, 0x0)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_poll_runtime.go:83 +0x32

internal/poll.(*pollDesc).waitRead(...)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_poll_runtime.go:88

internal/poll.(*FD).Read(0xc0000ac480, {0xc0000de000, 0x1000, 0x1000})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_unix.go:167 +0x25a

net.(*netFD).Read(0xc0000ac480, {0xc0000de000?, 0xc002e3cf90?, 0x4d7d00?})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/fd_posix.go:55 +0x29

net.(*conn).Read(0xc0000100d8, {0xc0000de000?, 0x133?, 0xc00008fbb8?})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/net.go:183 +0x45

net/http.(*persistConn).Read(0xc0000b2360, {0xc0000de000?, 0x181?, 0xc000238130?})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:1929 +0x4e

bufio.(*Reader).fill(0xc0000a85a0)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/bufio/bufio.go:106 +0x103

bufio.(*Reader).ReadSlice(0xc0000a85a0, 0x81?)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/bufio/bufio.go:371 +0x2f

net/http/internal.readChunkLine(0x7cde?)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/internal/chunked.go:129 +0x25

net/http/internal.(*chunkedReader).beginChunk(0xc0000c5050)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/internal/chunked.go:48 +0x28

net/http/internal.(*chunkedReader).Read(0xc0000c5050, {0xc0007c8000?, 0x0?, 0xc00008fd68?})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/internal/chunked.go:98 +0x14e

net/http.(*body).readLocked(0xc000486600, {0xc0007c8000?, 0x1c31518?, 0x168bcc0?})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transfer.go:844 +0x3c

net/http.(*body).Read(0x0?, {0xc0007c8000?, 0x6f139e?, 0x0?})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transfer.go:836 +0x125

net/http.(*bodyEOFSignal).Read(0xc000486640, {0xc0007c8000, 0x7e00, 0x7e00})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:2774 +0x142

encoding/json.(*Decoder).refill(0xc0001783c0)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/encoding/json/stream.go:165 +0x17f

encoding/json.(*Decoder).readValue(0xc0001783c0)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/encoding/json/stream.go:140 +0xbb

encoding/json.(*Decoder).Decode(0xc0001783c0, {0x14dc460, 0xc0002380f0})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/encoding/json/stream.go:63 +0x78

github.com/couchbase/cbauth/metakv.doRunObserveChildren.func1()

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:284 +0x10b

created by github.com/couchbase/cbauth/metakv.doRunObserveChildren

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:280 +0x2eb

goroutine 69 [IO wait]:

internal/poll.runtime_pollWait(0x7fd36d9b2de8, 0x72)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/runtime/netpoll.go:302 +0x89

internal/poll.(*pollDesc).wait(0xc0000ac680?, 0xc0017db000?, 0x0)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_poll_runtime.go:83 +0x32

internal/poll.(*pollDesc).waitRead(...)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_poll_runtime.go:88

internal/poll.(*FD).Read(0xc0000ac680, {0xc0017db000, 0x1000, 0x1000})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_unix.go:167 +0x25a

net.(*netFD).Read(0xc0000ac680, {0xc0017db000?, 0x40d1c9?, 0x4?})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/fd_posix.go:55 +0x29

net.(*conn).Read(0xc0002083c0, {0xc0017db000?, 0x0?, 0x0?})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/net.go:183 +0x45

net/http.(*persistConn).Read(0xc00066dc20, {0xc0017db000?, 0xc00022ec00?, 0xc000227d30?})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:1929 +0x4e

bufio.(*Reader).fill(0xc0007885a0)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/bufio/bufio.go:106 +0x103

bufio.(*Reader).Peek(0xc0007885a0, 0x1)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/bufio/bufio.go:144 +0x5d

net/http.(*persistConn).readLoop(0xc00066dc20)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:2093 +0x1ac

created by net/http.(*Transport).dialConn

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:1750 +0x173e

goroutine 70 [select]:

net/http.(*persistConn).writeLoop(0xc00066dc20)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:2392 +0xf5

created by net/http.(*Transport).dialConn

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:1751 +0x1791

goroutine 82 [chan receive]:

github.com/couchbase/goutils/systemeventlog.(*SystemEventLoggerImpl).logEvents(0xc000674690)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goutils/systemeventlog/system_event_logger.go:186 +0xb7

created by github.com/couchbase/goutils/systemeventlog.NewSystemEventLogger

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goutils/systemeventlog/system_event_logger.go:125 +0x1d6

goroutine 267 [select]:

net/http.(*persistConn).writeLoop(0xc0005710e0)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:2392 +0xf5

created by net/http.(*Transport).dialConn

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:1751 +0x1791

goroutine 308 [select]:

github.com/couchbase/indexing/secondary/dcp.(*Client).runObserveStreamingEndpoint(0x18?, {0xc003f17200, 0x1a}, 0xc0028a8078, 0xc0028a8060, 0xc003e98600)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/dcp/pools.go:503 +0x6da

github.com/couchbase/indexing/secondary/dcp.(*Client).RunObserveCollectionManifestChanges(0xc003ea4000?, {0x17ec83f?, 0x0?}, {0xc003f1d8e0?, 0x0?}, 0x0?, 0x0?)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/dcp/pools.go:426 +0xc7

github.com/couchbase/indexing/secondary/common.(*serviceNotifierInstance).RunObserveCollectionManifestChanges(0xc003ea4000, {0xc003f1d8e0, 0x8})

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/services_notifier.go:165 +0x85

created by github.com/couchbase/indexing/secondary/common.(*serviceNotifierInstance).bucketsChangeCallback

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/services_notifier.go:137 +0x2a5

goroutine 307 [select]:

github.com/couchbase/indexing/secondary/dcp.(*Client).runObserveStreamingEndpoint(0x18?, {0xc003f171a0, 0x1a}, 0xc0028a8018, 0xc0028a8000, 0xc003e98600)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/dcp/pools.go:503 +0x6da

github.com/couchbase/indexing/secondary/dcp.(*Client).RunObserveCollectionManifestChanges(0xc003ea4000?, {0x17ec83f?, 0x0?}, {0xc003f1d8d8?, 0x0?}, 0x0?, 0x0?)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/dcp/pools.go:426 +0xc7

github.com/couchbase/indexing/secondary/common.(*serviceNotifierInstance).RunObserveCollectionManifestChanges(0xc003ea4000, {0xc003f1d8d8, 0x8})

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/services_notifier.go:165 +0x85

created by github.com/couchbase/indexing/secondary/common.(*serviceNotifierInstance).bucketsChangeCallback

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/services_notifier.go:137 +0x2a5

goroutine 72 [chan receive]:

github.com/couchbase/indexing/secondary/common.(*clusterInfoCacheLiteManager).handlePerBucketCollectionManifest(0xc002bba3c0, {0xc002cbca68, 0x7}, 0xc00066dc20?)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/cluster_info_lite.go:1246 +0x69

created by github.com/couchbase/indexing/secondary/common.newClusterInfoCacheLiteManager

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/cluster_info_lite.go:816 +0xbaa

goroutine 73 [chan receive]:

github.com/couchbase/indexing/secondary/common.(*clusterInfoCacheLiteManager).handleBucketInfoChangesPerBucket(0xc002bba3c0, {0xc002cbca68, 0x7}, 0x0?)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/cluster_info_lite.go:1353 +0x7c

created by github.com/couchbase/indexing/secondary/common.newClusterInfoCacheLiteManager

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/cluster_info_lite.go:822 +0xced