Details
-
Bug
-
Resolution: Fixed
-
Critical
-
6.5.0
Description
Reported on MB-30907.
It appears to be a deadlock in gometa. The following goroutine is blocking
1 @ 0x4347ea 0x4348de 0x444a5d 0x4446f0 0x6d7be0 0xd711d7 0xd7277c 0xd5d6e0 0xd5ba8e 0xd5b6b1 0xd5b18b 0xd5a955 0x4659a1
|
# 0x4446ef sync.runtime_Semacquire+0x2f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/sema.go:47
|
# 0x6d7bdf sync.(*Mutex).Lock+0xcf /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/sync/mutex.go:85
|
# 0xd711d6 github.com/couchbase/gometa/server.(*EmbeddedServer).UpdateStateOnCommit+0x2e6 goproj/src/github.com/couchbase/gometa/server/embeddedServer.go:768
|
# 0xd7277b github.com/couchbase/gometa/action.(*ServerAction).Commit+0x12b goproj/src/github.com/couchbase/gometa/action/action.go:174
|
# 0xd5d6df github.com/couchbase/gometa/protocol.(*Leader).commit+0xbf goproj/src/github.com/couchbase/gometa/protocol/leader.go:895
|
# 0xd5ba8d github.com/couchbase/gometa/protocol.(*Leader).newProposal+0x33d goproj/src/github.com/couchbase/gometa/protocol/leader.go:664
|
# 0xd5b6b0 github.com/couchbase/gometa/protocol.(*Leader).createProposal+0x300 goproj/src/github.com/couchbase/gometa/protocol/leader.go:622
|
# 0xd5b18a github.com/couchbase/gometa/protocol.(*Leader).handleMessage+0x28a goproj/src/github.com/couchbase/gometa/protocol/leader.go:584
|
# 0xd5a954 github.com/couchbase/gometa/protocol.(*Leader).listen+0x134 goproj/src/github.com/couchbase/gometa/protocol/leader.go:498
|
#
|
|
and others are waiting for it, causing the indexer main loop to block
1 @ 0x4347ea 0x4348de 0x444a5d 0x4446f0 0x6d8a8d 0xd6cefe 0xd6d29d 0xd6e6bf 0x9e59cc 0x4db0cf 0x4daffc 0x4d97f9 0x4d9383 0x4659a1
|
# 0x4446ef sync.runtime_Semacquire+0x2f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/sema.go:47
|
# 0x6d8a8c sync.(*RWMutex).RLock+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/sync/rwmutex.go:43
|
# 0xd6cefd github.com/couchbase/gometa/server.(*EmbeddedServer).isActiveNoLock+0x3d goproj/src/github.com/couchbase/gometa/server/embeddedServer.go:186
|
# 0xd6d29c github.com/couchbase/gometa/server.(*EmbeddedServer).postToIncomings+0x5c goproj/src/github.com/couchbase/gometa/server/embeddedServer.go:224
|
# 0xd6e6be github.com/couchbase/gometa/server.(*EmbeddedServer).MakeAsyncRequest+0x29e goproj/src/github.com/couchbase/gometa/server/embeddedServer.go:365
|
# 0x9e59cb github.com/couchbase/indexing/secondary/manager.(*IndexManager).NotifyStats+0xfb goproj/src/github.com/couchbase/indexing/secondary/manager/manager.go:644
|
# 0x4db0ce github.com/couchbase/indexing/secondary/indexer.(*clustMgrAgent).handleStatsInternal+0x9e goproj/src/github.com/couchbase/indexing/secondary/indexer/cluster_manager_agent.go:340
|
# 0x4daffb github.com/couchbase/indexing/secondary/indexer.(*clustMgrAgent).handleStats+0x7b goproj/src/github.com/couchbase/indexing/secondary/indexer/cluster_manager_agent.go:332
|
# 0x4d97f8 github.com/couchbase/indexing/secondary/indexer.(*clustMgrAgent).handleSupvervisorCommands+0x3a8 goproj/src/github.com/couchbase/indexing/secondary/indexer/cluster_manager_agent.go:163
|
# 0x4d9382 github.com/couchbase/indexing/secondary/indexer.(*clustMgrAgent).run+0x112 goproj/src/github.com/couchbase/indexing/secondary/indexer/cluster_manager_agent.go:115
|
|
|
1 @ 0x4347ea 0x4348de 0x444a5d 0x4446f0 0x6d8a8d 0xd6cefe 0xd6f2af 0x9eade2 0x9e7341 0x9e6f30 0x9f4849 0x9f461b 0xa05988 0x748934 0x749d1f 0x74a72d 0x7471b7 0x4659a1
|
# 0x4446ef sync.runtime_Semacquire+0x2f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/sema.go:47
|
# 0x6d8a8c sync.(*RWMutex).RLock+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/sync/rwmutex.go:43
|
# 0xd6cefd github.com/couchbase/gometa/server.(*EmbeddedServer).isActiveNoLock+0x3d goproj/src/github.com/couchbase/gometa/server/embeddedServer.go:186
|
# 0xd6f2ae github.com/couchbase/gometa/server.(*EmbeddedServer).GetConfigValue+0x9e goproj/src/github.com/couchbase/gometa/server/embeddedServer.go:449
|
# 0x9eade1 github.com/couchbase/indexing/secondary/manager.(*LocalRepoRef).getLocalValue+0x41 goproj/src/github.com/couchbase/indexing/secondary/manager/meta_repo.go:753
|
# 0x9e7340 github.com/couchbase/indexing/secondary/manager.(*MetadataRepo).GetLocalValue+0xb0 goproj/src/github.com/couchbase/indexing/secondary/manager/meta_repo.go:189
|
# 0x9e6f2f github.com/couchbase/indexing/secondary/manager.(*MetadataRepo).GetLocalIndexerId+0x3f goproj/src/github.com/couchbase/indexing/secondary/manager/meta_repo.go:155
|
# 0x9f4848 github.com/couchbase/indexing/secondary/manager.(*requestHandlerContext).getLocalIndexMetadata+0x118 goproj/src/github.com/couchbase/indexing/secondary/manager/request_handler.go:876
|
# 0x9f461a github.com/couchbase/indexing/secondary/manager.(*requestHandlerContext).handleLocalIndexMetadataRequest+0xba goproj/src/github.com/couchbase/indexing/secondary/manager/request_handler.go:862
|
# 0xa05987 github.com/couchbase/indexing/secondary/manager.(*requestHandlerContext).(github.com/couchbase/indexing/secondary/manager.handleLocalIndexMetadataRequest)-fm+0x47 goproj/src/github.com/couchbase/indexing/secondary/manager/request_handler.go:171
|
# 0x748933 net/http.HandlerFunc.ServeHTTP+0x43 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:1726
|
# 0x749d1e net/http.(*ServeMux).ServeHTTP+0x7e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:2022
|
# 0x74a72c net/http.serverHandler.ServeHTTP+0x7c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:2202
|
# 0x7471b6 net/http.(*conn).serve+0x4b6 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:1579
|
|
|
1 @ 0x4347ea 0x4348de 0x444a5d 0x4446f0 0x6d8a8d 0xd6d30e 0xd6dccf 0x9ea820 0x9eafc5 0x9e8a2f 0x9d279b 0x4659a1
|
# 0x4446ef sync.runtime_Semacquire+0x2f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/sema.go:47
|
# 0x6d8a8c sync.(*RWMutex).RLock+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/sync/rwmutex.go:43
|
# 0xd6d30d github.com/couchbase/gometa/server.(*EmbeddedServer).postToIncomings+0xcd goproj/src/github.com/couchbase/gometa/server/embeddedServer.go:241
|
# 0xd6dcce github.com/couchbase/gometa/server.(*EmbeddedServer).Broadcast+0x2fe goproj/src/github.com/couchbase/gometa/server/embeddedServer.go:305
|
# 0x9ea81f github.com/couchbase/indexing/secondary/manager.(*LocalRepoRef).broadcast+0x6f goproj/src/github.com/couchbase/indexing/secondary/manager/meta_repo.go:689
|
# 0x9eafc4 github.com/couchbase/indexing/secondary/manager.(*MetadataRepo).broadcast+0x64 goproj/src/github.com/couchbase/indexing/secondary/manager/meta_repo.go:935
|
# 0x9e8a2e github.com/couchbase/indexing/secondary/manager.(*MetadataRepo).BroadcastIndexStats+0x7e goproj/src/github.com/couchbase/indexing/secondary/manager/meta_repo.go:427
|
# 0x9d279a github.com/couchbase/indexing/secondary/manager.(*LifecycleMgr).broadcastStats+0x1ca goproj/src/github.com/couchbase/indexing/secondary/manager/lifecycle.go:2045
|