Details
-
Bug
-
Resolution: Fixed
-
Critical
-
None
-
Untriaged
-
0
-
Unknown
Description
The stack trace is very long, so I've only pasted a partial excerpt. Rest can be found from the indexer.log.
Seen on 172.23.97.100
fatal error: concurrent map read and map write
|
|
goroutine 29577 [running]: |
runtime.throw({0x181dfc0?, 0xc016affbc0?}) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/runtime/panic.go:992 +0x71 fp=0xc0059246b0 sp=0xc005924680 pc=0x43f7d1 |
runtime.mapaccess1_faststr(0x159a220?, 0xc0000c4a80?, {0xc002e160f0, 0x7}) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/runtime/map_faststr.go:22 +0x3a5 fp=0xc005924718 sp=0xc0059246b0 pc=0x41a0a5 |
github.com/couchbase/indexing/secondary/indexer.(*indexer).ValidateKeyspace(0xc000124000, 0x4ae0?, {0xc002e160f0, 0x7}, {0xc02a7bf200, 0xb, 0x10}) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:11172 +0xa8 fp=0xc0059247d8 sp=0xc005924718 pc=0x129fe88 |
github.com/couchbase/indexing/secondary/indexer.(*indexer).startKeyspaceIdStream.func1(0xc02ab52840?) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:7335 +0x1fa fp=0xc005924fc8 sp=0xc0059247d8 pc=0x128593a |
github.com/couchbase/indexing/secondary/indexer.(*indexer).startKeyspaceIdStream.func2()
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:7510 +0x2a fp=0xc005924fe0 sp=0xc005924fc8 pc=0x128570a |
runtime.goexit()
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/runtime/asm_amd64.s:1571 +0x1 fp=0xc005924fe8 sp=0xc005924fe0 pc=0x472501 |
created by github.com/couchbase/indexing/secondary/indexer.(*indexer).startKeyspaceIdStream
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:7328 +0x1b7b |
|
goroutine 1 [runnable]: |
github.com/couchbase/indexing/secondary/indexer.(*indexer).addToInstsPerCollMap(0xc000124000, {0xc02b0a8000, 0xb, 0x8?}) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:11389 +0x210 |
github.com/couchbase/indexing/secondary/indexer.(*indexer).startKeyspaceIdStream(0xc000124000, 0x1, {0xc002dd8c90, 0x8}, 0xc01a6da000, 0x0, {0x0, 0x0, 0x0}, 0xc01d4cf500, ...) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:7274 +0x1034 |
github.com/couchbase/indexing/secondary/indexer.(*indexer).startStreams(0xc000124000) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:9073 +0x165 |
github.com/couchbase/indexing/secondary/indexer.(*indexer).bootstrap2(0xc000124000) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:7994 +0x69a |
github.com/couchbase/indexing/secondary/indexer.(*indexer).handleStorageWarmupDone(0xc000124000, {0x1c192e0?, 0xc00feb8cd8?}) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:7867 +0x449 |
github.com/couchbase/indexing/secondary/indexer.(*indexer).handleWorkerMsgs(0xc000124000, {0x1c192e0, 0xc00feb8cd8}) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:1519 +0x128f |
github.com/couchbase/indexing/secondary/indexer.(*indexer).run(0xc000124000) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:1128 +0x36e |
github.com/couchbase/indexing/secondary/indexer.NewIndexer(0x2b6c710?) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:554 +0x2989 |
main.main()
|
goproj/src/github.com/couchbase/indexing/secondary/cmd/indexer/main.go:206 +0x11d3 |
|
goroutine 21 [select]: |
github.com/couchbase/cbauth/cbauthimpl.(*tlsNotifier).loop(0xc00012c6a8) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/cbauthimpl/impl.go:405 +0x67 |
created by github.com/couchbase/cbauth/cbauthimpl.NewSVCForTest
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/cbauthimpl/impl.go:565 +0x37a |
|
goroutine 22 [select]: |
github.com/couchbase/cbauth/cbauthimpl.(*cfgChangeNotifier).loop(0xc00012c6c0) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/cbauthimpl/impl.go:325 +0x85 |
created by github.com/couchbase/cbauth/cbauthimpl.NewSVCForTest
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/cbauthimpl/impl.go:566 +0x3ca |
|
goroutine 23 [IO wait]: |
internal/poll.runtime_pollWait(0x7fd36d9b2fc8, 0x72) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/runtime/netpoll.go:302 +0x89 |
internal/poll.(*pollDesc).wait(0xc0000ac100?, 0xc0000bd000?, 0x0) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_poll_runtime.go:83 +0x32 |
internal/poll.(*pollDesc).waitRead(...)
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_poll_runtime.go:88 |
internal/poll.(*FD).Read(0xc0000ac100, {0xc0000bd000, 0x1000, 0x1000}) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_unix.go:167 +0x25a |
net.(*netFD).Read(0xc0000ac100, {0xc0000bd000?, 0xc00038a010?, 0xc00013f958?}) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/fd_posix.go:55 +0x29 |
net.(*conn).Read(0xc000010020, {0xc0000bd000?, 0x11?, 0xc00014d9e8?}) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/net.go:183 +0x45 |
bufio.(*Reader).Read(0xc0000a8120, {0xc000382001, 0xdff, 0x456ef4?}) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/bufio/bufio.go:236 +0x1b4 |
github.com/couchbase/cbauth/revrpc.(*minirwc).Read(0x191?, {0xc000382001?, 0x8?, 0x20?}) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/revrpc/revrpc.go:103 +0x25 |
encoding/json.(*Decoder).refill(0xc0000ca000) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/encoding/json/stream.go:165 +0x17f |
encoding/json.(*Decoder).readValue(0xc0000ca000) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/encoding/json/stream.go:140 +0xbb |
encoding/json.(*Decoder).Decode(0xc0000ca000, {0x158a3a0, 0xc0000a8260}) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/encoding/json/stream.go:63 +0x78 |
net/rpc/jsonrpc.(*serverCodec).ReadRequestHeader(0xc0000a8240, 0xc00007c0a0) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/rpc/jsonrpc/server.go:66 +0x85 |
net/rpc.(*Server).readRequestHeader(0xc0000c6000, {0x1c274c0, 0xc00007a1d0}) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/rpc/server.go:587 +0x66 |
net/rpc.(*Server).readRequest(0x0?, {0x1c274c0, 0xc00007a1d0}) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/rpc/server.go:547 +0x3b |
net/rpc.(*Server).ServeCodec(0xc0000c6000, {0x1c274c0?, 0xc00007a1d0}) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/rpc/server.go:462 +0x87 |
github.com/couchbase/cbauth/revrpc.(*Service).Run(0xc000071f60?, 0xc00008dfa0) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/revrpc/revrpc.go:236 +0x77c |
github.com/couchbase/cbauth/revrpc.BabysitService(0x0?, 0x0?, {0x1c1ae00?, 0xc00000e060?}) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/revrpc/revrpc.go:335 +0x58 |
github.com/couchbase/cbauth.runRPCForSvc(0x0?, 0xc0001901c0) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/default.go:57 +0xbd |
github.com/couchbase/cbauth.startDefault.func1()
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/default.go:66 +0x25 |
created by github.com/couchbase/cbauth.startDefault
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/default.go:65 +0xf9 |
|
goroutine 240 [select]: |
github.com/couchbase/indexing/secondary/dcp.(*Client).runObserveStreamingEndpoint(0xc000003520?, {0xc000556528, 0x17}, 0xc000542798, 0xc000542780, 0xc003e98600) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/dcp/pools.go:503 +0x6da |
github.com/couchbase/indexing/secondary/dcp.(*Client).RunObservePool(0xc003ea4000?, {0x17ec83f?, 0xc002b7b920?}, 0xc002c58000?, 0x163d840?) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/dcp/pools.go:395 +0xb2 |
github.com/couchbase/indexing/secondary/common.(*serviceNotifierInstance).RunPoolObserver(0xc003ea4000) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/services_notifier.go:147 +0x5c |
created by github.com/couchbase/indexing/secondary/common.NewServicesChangeNotifier
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/services_notifier.go:256 +0x50a |
|
goroutine 9 [chan receive]: |
github.com/couchbase/plasma.(*smrManager).run(0xc000669800) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/smr.go:526 +0x96 |
created by github.com/couchbase/plasma.NewSmrManager
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/smr.go:435 +0xaf |
|
goroutine 7 [select]: |
net/http.(*persistConn).readLoop(0xc0000b2360) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:2213 +0xd99 |
created by net/http.(*Transport).dialConn
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:1750 +0x173e |
|
goroutine 8 [select]: |
net/http.(*persistConn).writeLoop(0xc0000b2360) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:2392 +0xf5 |
created by net/http.(*Transport).dialConn
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:1751 +0x1791 |
|
goroutine 24 [chan receive]: |
github.com/couchbase/regulator/utils/logging.duplicateFlushLoop()
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/regulator/utils/logging/logger_golog.go:253 +0x25 |
created by github.com/couchbase/regulator/utils/logging.init.0 |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/regulator/utils/logging/logger_golog.go:49 +0x7e |
|
goroutine 10 [chan receive]: |
github.com/couchbase/plasma.runCleanerAutoTuner()
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/auto_tuner.go:653 +0x194 |
created by github.com/couchbase/plasma.init.2 |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/shard.go:211 +0x1fa |
|
goroutine 11 [chan receive]: |
github.com/couchbase/plasma.singletonWorker()
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/shard.go:4871 +0xb0 |
created by github.com/couchbase/plasma.init.2 |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/shard.go:212 +0x206 |
|
goroutine 12 [chan receive]: |
github.com/couchbase/plasma.systemResourceTracker()
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/mem.go:541 +0xa5 |
created by github.com/couchbase/plasma.init.2 |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/shard.go:213 +0x212 |
|
goroutine 13 [chan receive]: |
github.com/couchbase/plasma.AggregateAndLogStats()
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/shard.go:4832 +0x147 |
created by github.com/couchbase/plasma.init.2 |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/shard.go:214 +0x21e |
|
goroutine 32 [chan receive]: |
github.com/couchbase/plasma.(*CleanerAutoTuner).refreshCleanerBandwidth(0xc0003d8100) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/auto_tuner.go:664 +0x86 |
created by github.com/couchbase/plasma.runCleanerAutoTuner
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/auto_tuner.go:650 +0x136 |
|
goroutine 14 [chan receive]: |
github.com/couchbase/plasma.(*TenantMgr).Run(0xc00020b360) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/tenant.go:419 +0x9d |
created by github.com/couchbase/plasma.init.3 |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/tenant.go:57 +0x5d |
|
goroutine 51 [syscall]: |
syscall.Syscall(0x0, 0x0, 0xc0003c27ac, 0x4) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/syscall/asm_linux_amd64.s:20 +0x5 |
syscall.read(0xc000112060?, {0xc0003c27ac?, 0x3?, 0x10000c0000021a0?}) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/syscall/zsyscall_linux_amd64.go:696 +0x4d |
syscall.Read(...)
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/syscall/syscall_unix.go:188 |
internal/poll.ignoringEINTRIO(...)
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_unix.go:794 |
internal/poll.(*FD).Read(0xc000112060?, {0xc0003c27ac?, 0x4?, 0x4?}) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_unix.go:163 +0x285 |
os.(*File).read(...)
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/os/file_posix.go:31 |
os.(*File).Read(0xc000134000, {0xc0003c27ac?, 0xc0001fa930?, 0x18be388?}) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/os/file.go:119 +0x5e |
github.com/couchbase/indexing/secondary/common.ExitOnStdinClose()
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/util.go:440 +0x39 |
created by main.main
|
goproj/src/github.com/couchbase/indexing/secondary/cmd/indexer/main.go:110 +0xb9b |
|
goroutine 16 [select]: |
github.com/couchbase/cbauth/metakv.doRunObserveChildren(0xc0001c2420?, {0x1c14610, 0x1}, 0xc00013fe88, 0xc00022e7e0) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:301 +0x429 |
github.com/couchbase/cbauth/metakv.(*store).runObserveChildren(...)
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:259 |
github.com/couchbase/cbauth/metakv.RunObserveChildren({0x1c14610?, 0x0?}, 0x0?, 0x0?) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:389 +0x58 |
github.com/couchbase/indexing/secondary/indexer.NewSettingsManager.func1.1(0x0?, {0x0?, 0x0?}) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/settings.go:103 +0xcc |
github.com/couchbase/indexing/secondary/common.(*RetryHelper).Run(0xc00013ffa8) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/retry_helper.go:36 +0x83 |
github.com/couchbase/indexing/secondary/indexer.NewSettingsManager.func1()
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/settings.go:107 +0x75 |
created by github.com/couchbase/indexing/secondary/indexer.NewSettingsManager
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/settings.go:98 +0x54a |
|
goroutine 66 [chan receive]: |
github.com/couchbase/indexing/secondary/indexer.(*settingsManager).run(0xc000486540) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/settings.go:315 +0x4e |
created by github.com/couchbase/indexing/secondary/indexer.NewSettingsManager
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/settings.go:114 +0x58d |
|
goroutine 71 [chan receive]: |
github.com/couchbase/indexing/secondary/common.(*clusterInfoCacheLiteManager).handlePoolsChangeNotifications(0xc002bba3c0) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/cluster_info_lite.go:1079 +0x66 |
created by github.com/couchbase/indexing/secondary/common.newClusterInfoCacheLiteManager
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/cluster_info_lite.go:808 +0x893 |
|
goroutine 33 [IO wait]: |
internal/poll.runtime_pollWait(0x7fd36d9b2ed8, 0x72) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/runtime/netpoll.go:302 +0x89 |
internal/poll.(*pollDesc).wait(0xc0000ac480?, 0xc0000de000?, 0x0) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_poll_runtime.go:83 +0x32 |
internal/poll.(*pollDesc).waitRead(...)
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_poll_runtime.go:88 |
internal/poll.(*FD).Read(0xc0000ac480, {0xc0000de000, 0x1000, 0x1000}) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_unix.go:167 +0x25a |
net.(*netFD).Read(0xc0000ac480, {0xc0000de000?, 0xc002e3cf90?, 0x4d7d00?}) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/fd_posix.go:55 +0x29 |
net.(*conn).Read(0xc0000100d8, {0xc0000de000?, 0x133?, 0xc00008fbb8?}) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/net.go:183 +0x45 |
net/http.(*persistConn).Read(0xc0000b2360, {0xc0000de000?, 0x181?, 0xc000238130?}) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:1929 +0x4e |
bufio.(*Reader).fill(0xc0000a85a0) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/bufio/bufio.go:106 +0x103 |
bufio.(*Reader).ReadSlice(0xc0000a85a0, 0x81?) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/bufio/bufio.go:371 +0x2f |
net/http/internal.readChunkLine(0x7cde?) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/internal/chunked.go:129 +0x25 |
net/http/internal.(*chunkedReader).beginChunk(0xc0000c5050) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/internal/chunked.go:48 +0x28 |
net/http/internal.(*chunkedReader).Read(0xc0000c5050, {0xc0007c8000?, 0x0?, 0xc00008fd68?}) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/internal/chunked.go:98 +0x14e |
net/http.(*body).readLocked(0xc000486600, {0xc0007c8000?, 0x1c31518?, 0x168bcc0?}) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transfer.go:844 +0x3c |
net/http.(*body).Read(0x0?, {0xc0007c8000?, 0x6f139e?, 0x0?}) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transfer.go:836 +0x125 |
net/http.(*bodyEOFSignal).Read(0xc000486640, {0xc0007c8000, 0x7e00, 0x7e00}) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:2774 +0x142 |
encoding/json.(*Decoder).refill(0xc0001783c0) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/encoding/json/stream.go:165 +0x17f |
encoding/json.(*Decoder).readValue(0xc0001783c0) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/encoding/json/stream.go:140 +0xbb |
encoding/json.(*Decoder).Decode(0xc0001783c0, {0x14dc460, 0xc0002380f0}) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/encoding/json/stream.go:63 +0x78 |
github.com/couchbase/cbauth/metakv.doRunObserveChildren.func1()
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:284 +0x10b |
created by github.com/couchbase/cbauth/metakv.doRunObserveChildren
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:280 +0x2eb |
|
goroutine 69 [IO wait]: |
internal/poll.runtime_pollWait(0x7fd36d9b2de8, 0x72) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/runtime/netpoll.go:302 +0x89 |
internal/poll.(*pollDesc).wait(0xc0000ac680?, 0xc0017db000?, 0x0) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_poll_runtime.go:83 +0x32 |
internal/poll.(*pollDesc).waitRead(...)
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_poll_runtime.go:88 |
internal/poll.(*FD).Read(0xc0000ac680, {0xc0017db000, 0x1000, 0x1000}) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_unix.go:167 +0x25a |
net.(*netFD).Read(0xc0000ac680, {0xc0017db000?, 0x40d1c9?, 0x4?}) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/fd_posix.go:55 +0x29 |
net.(*conn).Read(0xc0002083c0, {0xc0017db000?, 0x0?, 0x0?}) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/net.go:183 +0x45 |
net/http.(*persistConn).Read(0xc00066dc20, {0xc0017db000?, 0xc00022ec00?, 0xc000227d30?}) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:1929 +0x4e |
bufio.(*Reader).fill(0xc0007885a0) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/bufio/bufio.go:106 +0x103 |
bufio.(*Reader).Peek(0xc0007885a0, 0x1) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/bufio/bufio.go:144 +0x5d |
net/http.(*persistConn).readLoop(0xc00066dc20) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:2093 +0x1ac |
created by net/http.(*Transport).dialConn
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:1750 +0x173e |
|
goroutine 70 [select]: |
net/http.(*persistConn).writeLoop(0xc00066dc20) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:2392 +0xf5 |
created by net/http.(*Transport).dialConn
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:1751 +0x1791 |
|
goroutine 82 [chan receive]: |
github.com/couchbase/goutils/systemeventlog.(*SystemEventLoggerImpl).logEvents(0xc000674690) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goutils/systemeventlog/system_event_logger.go:186 +0xb7 |
created by github.com/couchbase/goutils/systemeventlog.NewSystemEventLogger
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goutils/systemeventlog/system_event_logger.go:125 +0x1d6 |
|
goroutine 267 [select]: |
net/http.(*persistConn).writeLoop(0xc0005710e0) |
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:2392 +0xf5 |
created by net/http.(*Transport).dialConn
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:1751 +0x1791 |
|
goroutine 308 [select]: |
github.com/couchbase/indexing/secondary/dcp.(*Client).runObserveStreamingEndpoint(0x18?, {0xc003f17200, 0x1a}, 0xc0028a8078, 0xc0028a8060, 0xc003e98600) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/dcp/pools.go:503 +0x6da |
github.com/couchbase/indexing/secondary/dcp.(*Client).RunObserveCollectionManifestChanges(0xc003ea4000?, {0x17ec83f?, 0x0?}, {0xc003f1d8e0?, 0x0?}, 0x0?, 0x0?) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/dcp/pools.go:426 +0xc7 |
github.com/couchbase/indexing/secondary/common.(*serviceNotifierInstance).RunObserveCollectionManifestChanges(0xc003ea4000, {0xc003f1d8e0, 0x8}) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/services_notifier.go:165 +0x85 |
created by github.com/couchbase/indexing/secondary/common.(*serviceNotifierInstance).bucketsChangeCallback
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/services_notifier.go:137 +0x2a5 |
|
goroutine 307 [select]: |
github.com/couchbase/indexing/secondary/dcp.(*Client).runObserveStreamingEndpoint(0x18?, {0xc003f171a0, 0x1a}, 0xc0028a8018, 0xc0028a8000, 0xc003e98600) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/dcp/pools.go:503 +0x6da |
github.com/couchbase/indexing/secondary/dcp.(*Client).RunObserveCollectionManifestChanges(0xc003ea4000?, {0x17ec83f?, 0x0?}, {0xc003f1d8d8?, 0x0?}, 0x0?, 0x0?) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/dcp/pools.go:426 +0xc7 |
github.com/couchbase/indexing/secondary/common.(*serviceNotifierInstance).RunObserveCollectionManifestChanges(0xc003ea4000, {0xc003f1d8d8, 0x8}) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/services_notifier.go:165 +0x85 |
created by github.com/couchbase/indexing/secondary/common.(*serviceNotifierInstance).bucketsChangeCallback
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/services_notifier.go:137 +0x2a5 |
|
goroutine 72 [chan receive]: |
github.com/couchbase/indexing/secondary/common.(*clusterInfoCacheLiteManager).handlePerBucketCollectionManifest(0xc002bba3c0, {0xc002cbca68, 0x7}, 0xc00066dc20?) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/cluster_info_lite.go:1246 +0x69 |
created by github.com/couchbase/indexing/secondary/common.newClusterInfoCacheLiteManager
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/cluster_info_lite.go:816 +0xbaa |
|
goroutine 73 [chan receive]: |
github.com/couchbase/indexing/secondary/common.(*clusterInfoCacheLiteManager).handleBucketInfoChangesPerBucket(0xc002bba3c0, {0xc002cbca68, 0x7}, 0x0?) |
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/cluster_info_lite.go:1353 +0x7c |
created by github.com/couchbase/indexing/secondary/common.newClusterInfoCacheLiteManager
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/cluster_info_lite.go:822 +0xced |
cbcollect logs ->
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1673327127/collectinfo-2023-01-10T050529-ns_1%40172.23.105.122.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1673327127/collectinfo-2023-01-10T050529-ns_1%40172.23.106.171.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1673327127/collectinfo-2023-01-10T050529-ns_1%40172.23.106.176.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1673327127/collectinfo-2023-01-10T050529-ns_1%40172.23.106.30.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1673327127/collectinfo-2023-01-10T050529-ns_1%40172.23.96.230.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1673327127/collectinfo-2023-01-10T050529-ns_1%40172.23.96.245.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1673327127/collectinfo-2023-01-10T050529-ns_1%40172.23.97.100.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1673327127/collectinfo-2023-01-10T050529-ns_1%40172.23.97.108.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1673327127/collectinfo-2023-01-10T050529-ns_1%40172.23.97.109.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1673327127/collectinfo-2023-01-10T050529-ns_1%40172.23.97.66.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1673327127/collectinfo-2023-01-10T050529-ns_1%40172.23.97.67.zip
Attachments
Issue Links
- relates to
-
MB-55237 [BP to 7.1.4] [System Test] Panic seen in indexer
- Closed