============================================================================== Indexer Go routine dump: ['curl', '-sS', '--proxy', '', '-K-', 'http://127.0.0.1:9102/debug/pprof/goroutine?debug=1'] ============================================================================== goroutine profile: total 100935 64255 @ 0x43487a 0x43496e 0x444aed 0x444780 0x6bd8f0 0xcb238a 0xcb752d 0x4659c1 # 0x44477f sync.runtime_Semacquire+0x2f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/sema.go:47 # 0x6bd8ef sync.(*Mutex).Lock+0xcf /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/sync/mutex.go:85 # 0xcb2389 github.com/couchbase/gometa/protocol.(*Leader).AddObserver+0x39 goproj/src/github.com/couchbase/gometa/protocol/leader.go:326 # 0xcb752c github.com/couchbase/gometa/protocol.(*LeaderServer).startProxy+0x3ec goproj/src/github.com/couchbase/gometa/protocol/leaderServer.go:244 4978 @ 0x43487a 0x43496e 0x40c39f 0x40c0a5 0xc984c5 0x4659c1 # 0xc984c4 github.com/couchbase/gometa/common.(*PeerPipe).doSend+0xa4 goproj/src/github.com/couchbase/gometa/common/peerPipe.go:155 4977 @ 0x43487a 0x42f13b 0x42e899 0x7a53c8 0x7a5434 0x7a6c71 0x7b7d00 0xc99a02 0xc98ea4 0x4659c1 # 0x42e898 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53c7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5433 net.(*pollDesc).waitRead+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:78 # 0x7a6c70 net.(*netFD).Read+0x1a0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:243 # 0x7b7cff net.(*conn).Read+0x6f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/net.go:173 # 0xc99a01 github.com/couchbase/gometa/common.(*PeerPipe).readBytes+0xc1 goproj/src/github.com/couchbase/gometa/common/peerPipe.go:270 # 0xc98ea3 github.com/couchbase/gometa/common.(*PeerPipe).doReceive+0x93 goproj/src/github.com/couchbase/gometa/common/peerPipe.go:203 4804 @ 0x43487a 0x443f15 0x442cac 0x9dab61 0x4659c1 # 0x9dab60 github.com/couchbase/indexing/secondary/manager/client.(*watcher).timeoutChecker+0x150 goproj/src/github.com/couchbase/indexing/secondary/manager/client/metadata_provider.go:3915 4804 @ 0x43487a 0x443f15 0x442cac 0xcad70e 0x4659c1 # 0xcad70d github.com/couchbase/gometa/protocol.(*Follower).startListener+0x33d goproj/src/github.com/couchbase/gometa/protocol/follower.go:149 4804 @ 0x43487a 0x443f15 0x442cac 0xcba7a5 0xcb99f7 0xcb937f 0x4659c1 # 0xcba7a4 github.com/couchbase/gometa/protocol.runWatcher+0x5d4 goproj/src/github.com/couchbase/gometa/protocol/watcherServer.go:320 # 0xcb99f6 github.com/couchbase/gometa/protocol.runOnce+0x506 goproj/src/github.com/couchbase/gometa/protocol/watcherServer.go:176 # 0xcb937e github.com/couchbase/gometa/protocol.RunWatcherServerWithRequest+0x16e goproj/src/github.com/couchbase/gometa/protocol/watcherServer.go:49 4169 @ 0x43487a 0x443f15 0x442cac 0x9d85be 0x9cf98f 0x4659c1 # 0x9d85bd github.com/couchbase/indexing/secondary/manager/client.(*watcher).waitForReady+0x37d goproj/src/github.com/couchbase/indexing/secondary/manager/client/metadata_provider.go:3525 # 0x9cf98e github.com/couchbase/indexing/secondary/manager/client.(*MetadataProvider).retryHelper+0x20e goproj/src/github.com/couchbase/indexing/secondary/manager/client/metadata_provider.go:2285 4017 @ 0x43487a 0x443f15 0x442cac 0xcb94d4 0x4659c1 # 0xcb94d3 github.com/couchbase/gometa/protocol.RunWatcherServerWithRequest+0x2c3 goproj/src/github.com/couchbase/gometa/protocol/watcherServer.go:55 2967 @ 0x43487a 0x43496e 0x445182 0x6bd640 0x9da8a3 0x9da0bf 0x9d8666 0x9cf7ea 0x4659c1 # 0x445181 sync.runtime_notifyListWait+0x121 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/sema.go:267 # 0x6bd63f sync.(*Cond).Wait+0x7f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/sync/cond.go:57 # 0x9da8a2 github.com/couchbase/indexing/secondary/manager/client.(*watcher).makeRequest+0x322 goproj/src/github.com/couchbase/indexing/secondary/manager/client/metadata_provider.go:3897 # 0x9da0be github.com/couchbase/indexing/secondary/manager/client.(*watcher).refreshServiceMap+0xae goproj/src/github.com/couchbase/indexing/secondary/manager/client/metadata_provider.go:3825 # 0x9d8665 github.com/couchbase/indexing/secondary/manager/client.(*watcher).notifyReady+0x75 goproj/src/github.com/couchbase/indexing/secondary/manager/client/metadata_provider.go:3550 # 0x9cf7e9 github.com/couchbase/indexing/secondary/manager/client.(*MetadataProvider).retryHelper+0x69 goproj/src/github.com/couchbase/indexing/secondary/manager/client/metadata_provider.go:2292 186 @ 0x43487a 0x43496e 0x40c39f 0x40c0a5 0xa9d4ba 0x4659c1 # 0xa9d4b9 github.com/couchbase/plasma.(*Plasma).smrWorker+0x59 goproj/src/github.com/couchbase/plasma/smr.go:98 120 @ 0x43487a 0x43496e 0x40c39f 0x40c0a5 0xca8cf5 0xca7f8f 0xca79bb 0x4659c1 # 0xca8cf4 github.com/couchbase/gometa/protocol.listen+0x74 goproj/src/github.com/couchbase/gometa/protocol/discovery.go:1224 # 0xca7f8e github.com/couchbase/gometa/protocol.(*FollowerSyncProxy).receiveAndUpdateAcceptedEpoch+0x8e goproj/src/github.com/couchbase/gometa/protocol/discovery.go:1055 # 0xca79ba github.com/couchbase/gometa/protocol.(*FollowerSyncProxy).execute+0x26a goproj/src/github.com/couchbase/gometa/protocol/discovery.go:1001 120 @ 0x43487a 0x443f15 0x442cac 0xca74fe 0x4659c1 # 0xca74fd github.com/couchbase/gometa/protocol.(*FollowerSyncProxy).Start+0x44d goproj/src/github.com/couchbase/gometa/protocol/discovery.go:898 120 @ 0x43487a 0x443f15 0x442cac 0xcba095 0xcb981b 0xcb937f 0x4659c1 # 0xcba094 github.com/couchbase/gometa/protocol.syncWithPeer+0x5d4 goproj/src/github.com/couchbase/gometa/protocol/watcherServer.go:208 # 0xcb981a github.com/couchbase/gometa/protocol.runOnce+0x32a goproj/src/github.com/couchbase/gometa/protocol/watcherServer.go:172 # 0xcb937e github.com/couchbase/gometa/protocol.RunWatcherServerWithRequest+0x16e goproj/src/github.com/couchbase/gometa/protocol/watcherServer.go:49 69 @ 0x43487a 0x43496e 0x444aed 0x444780 0x6bd8f0 0xcb2caa 0xcbc223 0xc9ab97 0xcbc470 0xcb2ae0 0x4659c1 # 0x44477f sync.runtime_Semacquire+0x2f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/sema.go:47 # 0x6bd8ef sync.(*Mutex).Lock+0xcf /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/sync/mutex.go:85 # 0xcb2ca9 github.com/couchbase/gometa/protocol.(*Leader).removeListener+0x39 goproj/src/github.com/couchbase/gometa/protocol/leader.go:437 # 0xcbc222 github.com/couchbase/gometa/protocol.(*messageListener).start.func1.1+0x32 goproj/src/github.com/couchbase/gometa/protocol/leader.go:390 # 0xc9ab96 github.com/couchbase/gometa/common.SafeRun+0x56 goproj/src/github.com/couchbase/gometa/common/util.go:36 # 0xcbc46f github.com/couchbase/gometa/protocol.(*messageListener).start.func1+0x1ef goproj/src/github.com/couchbase/gometa/protocol/leader.go:391 # 0xcb2adf github.com/couchbase/gometa/protocol.(*messageListener).start+0x38f goproj/src/github.com/couchbase/gometa/protocol/leader.go:412 64 @ 0x43487a 0x443f15 0x442cac 0x571658 0x4659c1 # 0x571657 github.com/couchbase/indexing/secondary/indexer.(*plasmaSlice).handleCommandsWorker+0x727 goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:476 62 @ 0x43487a 0x43496e 0x44f9d1 0xa9f18f 0xaa56a4 0x4659c1 # 0x44f9d0 time.Sleep+0xe0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/time.go:59 # 0xa9f18e github.com/couchbase/plasma.(*DecayInterval).Sleep+0x2e goproj/src/github.com/couchbase/plasma/util.go:234 # 0xaa56a3 github.com/couchbase/plasma.(*Plasma).swapperDaemon.func1+0x133 goproj/src/github.com/couchbase/plasma/swapper.go:141 32 @ 0x43487a 0x42f13b 0x42e899 0x7a53c8 0x7a5494 0x7a5dda 0x7be247 0x7bddc9 0x7b5aa9 0x7c2d09 0x7c2c14 0x7c1390 0xcb04dd 0xcb957f 0xcb937f 0x4659c1 # 0x42e898 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53c7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5493 net.(*pollDesc).waitWrite+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:82 # 0x7a5dd9 net.(*netFD).connect+0x269 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:152 # 0x7be246 net.(*netFD).dial+0x136 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/sock_posix.go:137 # 0x7bddc8 net.socket+0x208 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/sock_posix.go:89 # 0x7b5aa8 net.internetSocket+0x128 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/ipsock_posix.go:158 # 0x7c2d08 net.doDialTCP+0xb8 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/tcpsock_posix.go:58 # 0x7c2c13 net.dialTCP+0xe3 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/tcpsock_posix.go:54 # 0x7c138f net.DialTCP+0x45f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/tcpsock.go:190 # 0xcb04dc github.com/couchbase/gometa/protocol.createConnection+0x8c goproj/src/github.com/couchbase/gometa/protocol/followerServer.go:223 # 0xcb957e github.com/couchbase/gometa/protocol.runOnce+0x8e goproj/src/github.com/couchbase/gometa/protocol/watcherServer.go:154 # 0xcb937e github.com/couchbase/gometa/protocol.RunWatcherServerWithRequest+0x16e goproj/src/github.com/couchbase/gometa/protocol/watcherServer.go:49 32 @ 0x43487a 0x443f15 0x442cac 0x6002e0 0x4659c1 # 0x6002df github.com/couchbase/indexing/secondary/indexer.(*streamWorker).start+0x23f goproj/src/github.com/couchbase/indexing/secondary/indexer/stream_reader.go:542 31 @ 0x43487a 0x43496e 0x40c39f 0x40c065 0xaa5767 0x4659c1 # 0xaa5766 github.com/couchbase/plasma.(*Plasma).swapperDaemon.func2+0x46 goproj/src/github.com/couchbase/plasma/swapper.go:148 31 @ 0x43487a 0x43496e 0x444aed 0x444780 0x6bedf7 0xa9e89b 0x4659c1 # 0x44477f sync.runtime_Semacquire+0x2f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/sema.go:47 # 0x6bedf6 sync.(*WaitGroup).Wait+0x96 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/sync/waitgroup.go:131 # 0xa9e89a github.com/couchbase/plasma.(*Plasma).swapperDaemon+0x17a goproj/src/github.com/couchbase/plasma/swapper.go:152 31 @ 0x43487a 0x43496e 0x44f9d1 0xa9580b 0x4659c1 # 0x44f9d0 time.Sleep+0xe0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/time.go:59 # 0xa9580a github.com/couchbase/plasma.(*Plasma).runtimeStats+0xca goproj/src/github.com/couchbase/plasma/plasma.go:634 31 @ 0x43487a 0x43496e 0x44f9d1 0xa964e1 0x4659c1 # 0x44f9d0 time.Sleep+0xe0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/time.go:59 # 0xa964e0 github.com/couchbase/plasma.(*Plasma).monitorStatistics+0x180 goproj/src/github.com/couchbase/plasma/plasma.go:796 31 @ 0x43487a 0x443f15 0x442cac 0xa7e31b 0x4659c1 # 0xa7e31a github.com/couchbase/plasma.(*Plasma).lssCleanerDaemon+0x22a goproj/src/github.com/couchbase/plasma/lss_cleaner.go:369 31 @ 0x43487a 0x443f15 0x442cac 0xa823ab 0x4659c1 # 0xa823aa github.com/couchbase/plasma.(*Plasma).mvccPurgerDaemon+0x47a goproj/src/github.com/couchbase/plasma/mvcc_purger.go:73 15 @ 0x43487a 0x443f15 0x442cac 0x7412ad 0x4659c1 # 0x7412ac net/http.(*persistConn).writeLoop+0x3bc /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transport.go:1646 14 @ 0x43487a 0x443f15 0x442cac 0xcb2c23 0x4659c1 # 0xcb2c22 github.com/couchbase/gometa/protocol.(*messageListener).start+0x4d2 goproj/src/github.com/couchbase/gometa/protocol/leader.go:403 8 @ 0x43487a 0x42f13b 0x42e899 0x7a53c8 0x7a5434 0x7a6c71 0x7b7d00 0x73eae4 0xc21eec 0xc220d2 0x73f701 0x4659c1 # 0x42e898 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53c7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5433 net.(*pollDesc).waitRead+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:78 # 0x7a6c70 net.(*netFD).Read+0x1a0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:243 # 0x7b7cff net.(*conn).Read+0x6f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/net.go:173 # 0x73eae3 net/http.(*persistConn).Read+0x153 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transport.go:1261 # 0xc21eeb bufio.(*Reader).fill+0x10b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:97 # 0xc220d1 bufio.(*Reader).Peek+0x61 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:129 # 0x73f700 net/http.(*persistConn).readLoop+0x1a0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transport.go:1418 8 @ 0x43487a 0x42f13b 0x42e899 0x7a53c8 0x7a5434 0x7a6c71 0x7b7d00 0xcd9a09 0xcda4ea 0xcd93a0 0xbc830c 0x4659c1 # 0x42e898 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53c7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5433 net.(*pollDesc).waitRead+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:78 # 0x7a6c70 net.(*netFD).Read+0x1a0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:243 # 0x7b7cff net.(*conn).Read+0x6f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/net.go:173 # 0xcd9a08 github.com/couchbase/indexing/secondary/transport.fullRead+0x88 goproj/src/github.com/couchbase/indexing/secondary/transport/transport.go:189 # 0xcda4e9 github.com/couchbase/indexing/secondary/transport.Receive+0x89 goproj/src/github.com/couchbase/indexing/secondary/transport/util.go:60 # 0xcd939f github.com/couchbase/indexing/secondary/transport.(*TransportPacket).Receive+0x6f goproj/src/github.com/couchbase/indexing/secondary/transport/transport.go:119 # 0xbc830b github.com/couchbase/indexing/secondary/queryport.(*Server).doReceive+0x3bb goproj/src/github.com/couchbase/indexing/secondary/queryport/server.go:225 8 @ 0x43487a 0x43496e 0x40c39f 0x40c0a5 0xbc7d73 0x4659c1 # 0xbc7d72 github.com/couchbase/indexing/secondary/queryport.(*Server).handleConnection+0x262 goproj/src/github.com/couchbase/indexing/secondary/queryport/server.go:197 8 @ 0x43487a 0x443f15 0x442cac 0xbc8905 0x4659c1 # 0xbc8904 github.com/couchbase/indexing/secondary/queryport.(*Server).doPing+0x1b4 goproj/src/github.com/couchbase/indexing/secondary/queryport/server.go:257 7 @ 0x43487a 0x43496e 0x40c39f 0x40c0a5 0x4879fc 0x4659c1 # 0x4879fb github.com/couchbase/indexing/secondary/common.(*vbSeqnosReader).Routine+0x7b goproj/src/github.com/couchbase/indexing/secondary/common/dcp_seqno.go:104 7 @ 0x43487a 0x443f15 0x442cac 0x73ff29 0x4659c1 # 0x73ff28 net/http.(*persistConn).readLoop+0x9c8 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transport.go:1541 6 @ 0x43487a 0x42f13b 0x42e899 0x7a53c8 0x7a5434 0x7a6c71 0x7b7d00 0x727a34 0xc21eec 0xc22b95 0xc22d17 0xc1e6be 0xc1e4bf 0x7224a5 0x728dad 0x72cde3 0x4659c1 # 0x42e898 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53c7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5433 net.(*pollDesc).waitRead+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:78 # 0x7a6c70 net.(*netFD).Read+0x1a0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:243 # 0x7b7cff net.(*conn).Read+0x6f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/net.go:173 # 0x727a33 net/http.(*connReader).Read+0x143 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:586 # 0xc21eeb bufio.(*Reader).fill+0x10b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:97 # 0xc22b94 bufio.(*Reader).ReadSlice+0xb4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:330 # 0xc22d16 bufio.(*Reader).ReadLine+0x36 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:359 # 0xc1e6bd net/textproto.(*Reader).readLineSlice+0x5d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/textproto/reader.go:55 # 0xc1e4be net/textproto.(*Reader).ReadLine+0x2e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/textproto/reader.go:36 # 0x7224a4 net/http.readRequest+0xa4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/request.go:793 # 0x728dac net/http.(*conn).readRequest+0x10c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:765 # 0x72cde2 net/http.(*conn).serve+0x3d2 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:1532 6 @ 0x43487a 0x443f15 0x442cac 0x65259c 0x4659c1 # 0x65259b github.com/couchbase/indexing/secondary/indexer.(*timekeeper).startTimer.func1+0x13b goproj/src/github.com/couchbase/indexing/secondary/indexer/timekeeper.go:2947 5 @ 0x43487a 0x42f13b 0x42e899 0x7a53c8 0x7a5434 0x7a6c71 0x7b7d00 0x73eae4 0xc21eec 0xc22b95 0xc33a94 0xc334d2 0xc3373b 0x736761 0x7366a6 0x742746 0x773a2a 0x773678 0x7731be 0x94326d 0x4659c1 # 0x42e898 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53c7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5433 net.(*pollDesc).waitRead+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:78 # 0x7a6c70 net.(*netFD).Read+0x1a0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:243 # 0x7b7cff net.(*conn).Read+0x6f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/net.go:173 # 0x73eae3 net/http.(*persistConn).Read+0x153 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transport.go:1261 # 0xc21eeb bufio.(*Reader).fill+0x10b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:97 # 0xc22b94 bufio.(*Reader).ReadSlice+0xb4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:330 # 0xc33a93 net/http/internal.readChunkLine+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/internal/chunked.go:110 # 0xc334d1 net/http/internal.(*chunkedReader).beginChunk+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/internal/chunked.go:47 # 0xc3373a net/http/internal.(*chunkedReader).Read+0x7a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/internal/chunked.go:77 # 0x736760 net/http.(*body).readLocked+0x60 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transfer.go:651 # 0x7366a5 net/http.(*body).Read+0xf5 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transfer.go:643 # 0x742745 net/http.(*bodyEOFSignal).Read+0x115 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transport.go:1974 # 0x773a29 encoding/json.(*Decoder).refill+0xf9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/encoding/json/stream.go:152 # 0x773677 encoding/json.(*Decoder).readValue+0x197 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/encoding/json/stream.go:128 # 0x7731bd encoding/json.(*Decoder).Decode+0x8d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/encoding/json/stream.go:57 # 0x94326c github.com/couchbase/cbauth/metakv.doRunObserveChildren.func1+0x11c godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:270 5 @ 0x43487a 0x443f15 0x442cac 0x9d877b 0x9cf7ea 0x4659c1 # 0x9d877a github.com/couchbase/indexing/secondary/manager/client.(*watcher).notifyReady+0x18a goproj/src/github.com/couchbase/indexing/secondary/manager/client/metadata_provider.go:3557 # 0x9cf7e9 github.com/couchbase/indexing/secondary/manager/client.(*MetadataProvider).retryHelper+0x69 goproj/src/github.com/couchbase/indexing/secondary/manager/client/metadata_provider.go:2292 4 @ 0x43487a 0x42f13b 0x42e899 0x7a53c8 0x7a5434 0x7a6c71 0x7b7d00 0xcd9a09 0xcda4ea 0xcd93a0 0xbd059e 0x4659c1 # 0x42e898 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53c7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5433 net.(*pollDesc).waitRead+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:78 # 0x7a6c70 net.(*netFD).Read+0x1a0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:243 # 0x7b7cff net.(*conn).Read+0x6f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/net.go:173 # 0xcd9a08 github.com/couchbase/indexing/secondary/transport.fullRead+0x88 goproj/src/github.com/couchbase/indexing/secondary/transport/transport.go:189 # 0xcda4e9 github.com/couchbase/indexing/secondary/transport.Receive+0x89 goproj/src/github.com/couchbase/indexing/secondary/transport/util.go:60 # 0xcd939f github.com/couchbase/indexing/secondary/transport.(*TransportPacket).Receive+0x6f goproj/src/github.com/couchbase/indexing/secondary/transport/transport.go:119 # 0xbd059d github.com/couchbase/indexing/secondary/dataport.doReceive+0x28d goproj/src/github.com/couchbase/indexing/secondary/dataport/server.go:563 3 @ 0x43487a 0x443f15 0x442cac 0x940659 0x94003c 0x940a85 0x9e3a61 0x4946d2 0x9e3c74 0x4659c1 # 0x940658 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x5d8 godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0x94003b github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x5b godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0x940a84 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x9e3a60 github.com/couchbase/indexing/secondary/manager/common.(*CommandListener).ListenTokens.func2.1+0x70 goproj/src/github.com/couchbase/indexing/secondary/manager/common/token.go:658 # 0x4946d1 github.com/couchbase/indexing/secondary/common.(*RetryHelper).Run+0x51 goproj/src/github.com/couchbase/indexing/secondary/common/retry_helper.go:36 # 0x9e3c73 github.com/couchbase/indexing/secondary/manager/common.(*CommandListener).ListenTokens.func2+0x103 goproj/src/github.com/couchbase/indexing/secondary/manager/common/token.go:663 2 @ 0x43487a 0x443f15 0x442cac 0x9bb06f 0x4659c1 # 0x9bb06e github.com/couchbase/cbauth/service.withTimeout.func1+0x13e godeps/src/github.com/couchbase/cbauth/service/revrpc.go:38 1 @ 0x415f32 0x447cf7 0xd4a972 0x4659c1 # 0x447cf6 os/signal.signal_recv+0x156 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/sigqueue.go:116 # 0xd4a971 os/signal.loop+0x21 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/os/signal/signal_unix.go:22 1 @ 0x43487a 0x42f13b 0x42e899 0x7a53c8 0x7a5434 0x7a6c71 0x7b7d00 0x73eae4 0xc21eec 0xc22b95 0xc33a94 0xc334d2 0xc3373b 0x736761 0x7366a6 0x742746 0xc21eec 0xc22b95 0xc23026 0x82f191 0x82ed89 0x494c87 0x4659c1 # 0x42e898 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53c7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5433 net.(*pollDesc).waitRead+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:78 # 0x7a6c70 net.(*netFD).Read+0x1a0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:243 # 0x7b7cff net.(*conn).Read+0x6f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/net.go:173 # 0x73eae3 net/http.(*persistConn).Read+0x153 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transport.go:1261 # 0xc21eeb bufio.(*Reader).fill+0x10b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:97 # 0xc22b94 bufio.(*Reader).ReadSlice+0xb4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:330 # 0xc33a93 net/http/internal.readChunkLine+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/internal/chunked.go:110 # 0xc334d1 net/http/internal.(*chunkedReader).beginChunk+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/internal/chunked.go:47 # 0xc3373a net/http/internal.(*chunkedReader).Read+0x7a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/internal/chunked.go:77 # 0x736760 net/http.(*body).readLocked+0x60 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transfer.go:651 # 0x7366a5 net/http.(*body).Read+0xf5 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transfer.go:643 # 0x742745 net/http.(*bodyEOFSignal).Read+0x115 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transport.go:1974 # 0xc21eeb bufio.(*Reader).fill+0x10b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:97 # 0xc22b94 bufio.(*Reader).ReadSlice+0xb4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:330 # 0xc23025 bufio.(*Reader).ReadBytes+0x65 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:408 # 0x82f190 github.com/couchbase/indexing/secondary/dcp.(*Client).runObserveStreamingEndpoint+0x2f0 goproj/src/github.com/couchbase/indexing/secondary/dcp/pools.go:379 # 0x82ed88 github.com/couchbase/indexing/secondary/dcp.(*Client).RunObservePool+0x98 goproj/src/github.com/couchbase/indexing/secondary/dcp/pools.go:318 # 0x494c86 github.com/couchbase/indexing/secondary/common.(*serviceNotifierInstance).RunPoolObserver+0x96 goproj/src/github.com/couchbase/indexing/secondary/common/services_notifier.go:79 1 @ 0x43487a 0x42f13b 0x42e899 0x7a53c8 0x7a5434 0x7a6c71 0x7b7d00 0x73eae4 0xc21eec 0xc22b95 0xc33a94 0xc334d2 0xc3373b 0x736761 0x7366a6 0x742746 0xc21eec 0xc22b95 0xc23026 0x82f191 0x82ee6e 0x494e87 0x4659c1 # 0x42e898 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53c7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5433 net.(*pollDesc).waitRead+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:78 # 0x7a6c70 net.(*netFD).Read+0x1a0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:243 # 0x7b7cff net.(*conn).Read+0x6f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/net.go:173 # 0x73eae3 net/http.(*persistConn).Read+0x153 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transport.go:1261 # 0xc21eeb bufio.(*Reader).fill+0x10b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:97 # 0xc22b94 bufio.(*Reader).ReadSlice+0xb4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:330 # 0xc33a93 net/http/internal.readChunkLine+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/internal/chunked.go:110 # 0xc334d1 net/http/internal.(*chunkedReader).beginChunk+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/internal/chunked.go:47 # 0xc3373a net/http/internal.(*chunkedReader).Read+0x7a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/internal/chunked.go:77 # 0x736760 net/http.(*body).readLocked+0x60 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transfer.go:651 # 0x7366a5 net/http.(*body).Read+0xf5 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transfer.go:643 # 0x742745 net/http.(*bodyEOFSignal).Read+0x115 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transport.go:1974 # 0xc21eeb bufio.(*Reader).fill+0x10b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:97 # 0xc22b94 bufio.(*Reader).ReadSlice+0xb4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:330 # 0xc23025 bufio.(*Reader).ReadBytes+0x65 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:408 # 0x82f190 github.com/couchbase/indexing/secondary/dcp.(*Client).runObserveStreamingEndpoint+0x2f0 goproj/src/github.com/couchbase/indexing/secondary/dcp/pools.go:379 # 0x82ee6d github.com/couchbase/indexing/secondary/dcp.(*Client).RunObserveNodeServices+0xad goproj/src/github.com/couchbase/indexing/secondary/dcp/pools.go:331 # 0x494e86 github.com/couchbase/indexing/secondary/common.(*serviceNotifierInstance).RunServicesObserver+0x96 goproj/src/github.com/couchbase/indexing/secondary/common/services_notifier.go:88 1 @ 0x43487a 0x42f13b 0x42e899 0x7a53c8 0x7a5434 0x7a6c71 0x7b7d00 0xc21eec 0xc2254c 0x78861d 0x773a2a 0x773678 0x7731be 0xc4a5e4 0x77c96c 0x77c43a 0x77ba0a 0x788ded 0x7896fc 0x47b701 0x47ce95 0x4659c1 # 0x42e898 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53c7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5433 net.(*pollDesc).waitRead+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:78 # 0x7a6c70 net.(*netFD).Read+0x1a0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:243 # 0x7b7cff net.(*conn).Read+0x6f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/net.go:173 # 0xc21eeb bufio.(*Reader).fill+0x10b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:97 # 0xc2254b bufio.(*Reader).Read+0x1bb /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:209 # 0x78861c github.com/couchbase/cbauth/revrpc.(*minirwc).Read+0x4c godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:101 # 0x773a29 encoding/json.(*Decoder).refill+0xf9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/encoding/json/stream.go:152 # 0x773677 encoding/json.(*Decoder).readValue+0x197 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/encoding/json/stream.go:128 # 0x7731bd encoding/json.(*Decoder).Decode+0x8d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/encoding/json/stream.go:57 # 0xc4a5e3 net/rpc/jsonrpc.(*serverCodec).ReadRequestHeader+0x73 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/rpc/jsonrpc/server.go:66 # 0x77c96b net/rpc.(*Server).readRequestHeader+0x6b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/rpc/server.go:576 # 0x77c439 net/rpc.(*Server).readRequest+0x59 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/rpc/server.go:543 # 0x77ba09 net/rpc.(*Server).ServeCodec+0x89 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/rpc/server.go:462 # 0x788dec github.com/couchbase/cbauth/revrpc.(*Service).Run+0x79c godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:147 # 0x7896fb github.com/couchbase/cbauth/revrpc.BabysitService+0x5b godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:243 # 0x47b700 github.com/couchbase/cbauth.runRPCForSvc+0x100 godeps/src/github.com/couchbase/cbauth/default.go:59 # 0x47ce94 github.com/couchbase/cbauth.startDefault.func1+0x34 godeps/src/github.com/couchbase/cbauth/default.go:66 1 @ 0x43487a 0x42f13b 0x42e899 0x7a53c8 0x7a5434 0x7a6c71 0x7b7d00 0xc21eec 0xc2254c 0x78861d 0x773a2a 0x773678 0x7731be 0xc4a5e4 0x77c96c 0x77c43a 0x77ba0a 0x788ded 0x7896fc 0x9bae8e 0x5839b5 0x4659c1 # 0x42e898 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53c7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5433 net.(*pollDesc).waitRead+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:78 # 0x7a6c70 net.(*netFD).Read+0x1a0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:243 # 0x7b7cff net.(*conn).Read+0x6f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/net.go:173 # 0xc21eeb bufio.(*Reader).fill+0x10b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:97 # 0xc2254b bufio.(*Reader).Read+0x1bb /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:209 # 0x78861c github.com/couchbase/cbauth/revrpc.(*minirwc).Read+0x4c godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:101 # 0x773a29 encoding/json.(*Decoder).refill+0xf9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/encoding/json/stream.go:152 # 0x773677 encoding/json.(*Decoder).readValue+0x197 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/encoding/json/stream.go:128 # 0x7731bd encoding/json.(*Decoder).Decode+0x8d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/encoding/json/stream.go:57 # 0xc4a5e3 net/rpc/jsonrpc.(*serverCodec).ReadRequestHeader+0x73 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/rpc/jsonrpc/server.go:66 # 0x77c96b net/rpc.(*Server).readRequestHeader+0x6b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/rpc/server.go:576 # 0x77c439 net/rpc.(*Server).readRequest+0x59 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/rpc/server.go:543 # 0x77ba09 net/rpc.(*Server).ServeCodec+0x89 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/rpc/server.go:462 # 0x788dec github.com/couchbase/cbauth/revrpc.(*Service).Run+0x79c godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:147 # 0x7896fb github.com/couchbase/cbauth/revrpc.BabysitService+0x5b godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:243 # 0x9bae8d github.com/couchbase/cbauth/service.RegisterManager+0xbd godeps/src/github.com/couchbase/cbauth/service/revrpc.go:146 # 0x5839b4 github.com/couchbase/indexing/secondary/indexer.(*ServiceMgr).registerWithServer+0x1d4 goproj/src/github.com/couchbase/indexing/secondary/indexer/rebalance_service_manager.go:274 1 @ 0x43487a 0x42f13b 0x42e899 0x7a53c8 0x7a5434 0x7a8588 0x7c316e 0x7c1699 0x73177f 0x73099e 0x7306d4 0x62eb97 0x4659c1 # 0x42e898 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53c7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5433 net.(*pollDesc).waitRead+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:78 # 0x7a8587 net.(*netFD).accept+0x237 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:419 # 0x7c316d net.(*TCPListener).accept+0x2d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/tcpsock_posix.go:132 # 0x7c1698 net.(*TCPListener).AcceptTCP+0x48 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/tcpsock.go:209 # 0x73177e net/http.tcpKeepAliveListener.Accept+0x2e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:2608 # 0x73099d net/http.(*Server).Serve+0x1cd /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:2273 # 0x7306d3 net/http.(*Server).ListenAndServe+0xb3 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:2219 # 0x62eb96 github.com/couchbase/indexing/secondary/indexer.(*indexer).initHttpServer.func1+0x216 goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:373 1 @ 0x43487a 0x42f13b 0x42e899 0x7a53c8 0x7a5434 0x7a8588 0x7c316e 0x7c18b9 0x6e9cf7 0x73099e 0x7303e9 0x62f1d0 0x4659c1 # 0x42e898 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53c7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5433 net.(*pollDesc).waitRead+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:78 # 0x7a8587 net.(*netFD).accept+0x237 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:419 # 0x7c316d net.(*TCPListener).accept+0x2d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/tcpsock_posix.go:132 # 0x7c18b8 net.(*TCPListener).Accept+0x48 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/tcpsock.go:222 # 0x6e9cf6 crypto/tls.(*listener).Accept+0x36 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/crypto/tls/tls.go:52 # 0x73099d net/http.(*Server).Serve+0x1cd /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:2273 # 0x7303e8 net/http.Serve+0x98 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:2091 # 0x62f1cf github.com/couchbase/indexing/secondary/indexer.(*indexer).initHttpServer.func3+0x4ef goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:445 1 @ 0x43487a 0x42f13b 0x42e899 0x7a53c8 0x7a5434 0x7a8588 0x7c316e 0x7c18b9 0xbc780d 0x4659c1 # 0x42e898 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53c7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5433 net.(*pollDesc).waitRead+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:78 # 0x7a8587 net.(*netFD).accept+0x237 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:419 # 0x7c316d net.(*TCPListener).accept+0x2d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/tcpsock_posix.go:132 # 0x7c18b8 net.(*TCPListener).Accept+0x48 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/tcpsock.go:222 # 0xbc780c github.com/couchbase/indexing/secondary/queryport.(*Server).listener+0xac goproj/src/github.com/couchbase/indexing/secondary/queryport/server.go:150 1 @ 0x43487a 0x42f13b 0x42e899 0x7a53c8 0x7a5434 0x7a8588 0x7c316e 0x7c18b9 0xbcff4c 0x4659c1 # 0x42e898 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53c7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5433 net.(*pollDesc).waitRead+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:78 # 0x7a8587 net.(*netFD).accept+0x237 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:419 # 0x7c316d net.(*TCPListener).accept+0x2d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/tcpsock_posix.go:132 # 0x7c18b8 net.(*TCPListener).Accept+0x48 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/tcpsock.go:222 # 0xbcff4b github.com/couchbase/indexing/secondary/dataport.listener+0x4b goproj/src/github.com/couchbase/indexing/secondary/dataport/server.go:515 1 @ 0x43487a 0x42f13b 0x42e899 0x7a53c8 0x7a5434 0x7a8588 0x7c316e 0x7c18b9 0xc959b2 0x4659c1 # 0x42e898 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53c7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5433 net.(*pollDesc).waitRead+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:78 # 0x7a8587 net.(*netFD).accept+0x237 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:419 # 0x7c316d net.(*TCPListener).accept+0x2d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/tcpsock_posix.go:132 # 0x7c18b8 net.(*TCPListener).Accept+0x48 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/tcpsock.go:222 # 0xc959b1 github.com/couchbase/gometa/common.(*PeerListener).listen+0x61 goproj/src/github.com/couchbase/gometa/common/peerListener.go:135 1 @ 0x43487a 0x43496e 0x40b6a8 0x40b46d 0x50c3c6 0x509ddb 0x50732c 0x406e81 0x434414 0x4659c1 # 0x50c3c5 github.com/couchbase/indexing/secondary/indexer.(*indexer).handleWorkerMsgs+0x1be5 goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:854 # 0x509dda github.com/couchbase/indexing/secondary/indexer.(*indexer).run+0x17a goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:572 # 0x50732b github.com/couchbase/indexing/secondary/indexer.NewIndexer+0x1f2b goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:327 # 0x406e80 main.main+0x12e0 goproj/src/github.com/couchbase/indexing/secondary/cmd/indexer/main.go:165 # 0x434413 runtime.main+0x1f3 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/proc.go:183 1 @ 0x43487a 0x43496e 0x40b6a8 0x40b46d 0xc983ee 0xcb405e 0xcb3bc9 0xcb3881 0xcb335b 0xcb2e95 0x4659c1 # 0xc983ed github.com/couchbase/gometa/common.(*PeerPipe).Send+0xbd goproj/src/github.com/couchbase/gometa/common/peerPipe.go:129 # 0xcb405d github.com/couchbase/gometa/protocol.(*Leader).sendProposal+0x3bd goproj/src/github.com/couchbase/gometa/protocol/leader.go:642 # 0xcb3bc8 github.com/couchbase/gometa/protocol.(*Leader).newProposal+0x2a8 goproj/src/github.com/couchbase/gometa/protocol/leader.go:587 # 0xcb3880 github.com/couchbase/gometa/protocol.(*Leader).createProposal+0x300 goproj/src/github.com/couchbase/gometa/protocol/leader.go:554 # 0xcb335a github.com/couchbase/gometa/protocol.(*Leader).handleMessage+0x28a goproj/src/github.com/couchbase/gometa/protocol/leader.go:516 # 0xcb2e94 github.com/couchbase/gometa/protocol.(*Leader).listen+0x134 goproj/src/github.com/couchbase/gometa/protocol/leader.go:475 1 @ 0x43487a 0x43496e 0x40b6a8 0x40b46d 0xcb8533 0xcb6829 0xcc7196 0xcc68ff 0x4659c1 # 0xcb8532 github.com/couchbase/gometa/protocol.(*LeaderServer).processRequest+0x382 goproj/src/github.com/couchbase/gometa/protocol/leaderServer.go:391 # 0xcb6828 github.com/couchbase/gometa/protocol.RunLeaderServerWithCustomHandler+0x5c8 goproj/src/github.com/couchbase/gometa/protocol/leaderServer.go:130 # 0xcc7195 github.com/couchbase/gometa/server.(*EmbeddedServer).runOnce+0x165 goproj/src/github.com/couchbase/gometa/server/embeddedServer.go:483 # 0xcc68fe github.com/couchbase/gometa/server.(*EmbeddedServer).run+0x2e goproj/src/github.com/couchbase/gometa/server/embeddedServer.go:385 1 @ 0x43487a 0x43496e 0x40b6a8 0x40b46d 0xcc56a2 0x997f2c 0x4d3c5f 0x4d3b8c 0x4d258f 0x4d2173 0x4659c1 # 0xcc56a1 github.com/couchbase/gometa/server.(*EmbeddedServer).MakeAsyncRequest+0x2c1 goproj/src/github.com/couchbase/gometa/server/embeddedServer.go:246 # 0x997f2b github.com/couchbase/indexing/secondary/manager.(*IndexManager).NotifyStats+0xfb goproj/src/github.com/couchbase/indexing/secondary/manager/manager.go:617 # 0x4d3c5e github.com/couchbase/indexing/secondary/indexer.(*clustMgrAgent).handleStatsInternal+0x9e goproj/src/github.com/couchbase/indexing/secondary/indexer/cluster_manager_agent.go:308 # 0x4d3b8b github.com/couchbase/indexing/secondary/indexer.(*clustMgrAgent).handleStats+0x7b goproj/src/github.com/couchbase/indexing/secondary/indexer/cluster_manager_agent.go:300 # 0x4d258e github.com/couchbase/indexing/secondary/indexer.(*clustMgrAgent).handleSupvervisorCommands+0x34e goproj/src/github.com/couchbase/indexing/secondary/indexer/cluster_manager_agent.go:154 # 0x4d2172 github.com/couchbase/indexing/secondary/indexer.(*clustMgrAgent).run+0x112 goproj/src/github.com/couchbase/indexing/secondary/indexer/cluster_manager_agent.go:106 1 @ 0x43487a 0x43496e 0x40c39f 0x40c065 0x5f1709 0x5f1cfd 0x64c528 0x72e644 0x72fa2f 0x73051d 0x72cec7 0x4659c1 # 0x5f1708 github.com/couchbase/indexing/secondary/indexer.(*statsManager).getStorageStats+0x138 goproj/src/github.com/couchbase/indexing/secondary/indexer/stats_manager.go:1230 # 0x5f1cfc github.com/couchbase/indexing/secondary/indexer.(*statsManager).handleStorageStatsReq+0x14c goproj/src/github.com/couchbase/indexing/secondary/indexer/stats_manager.go:1257 # 0x64c527 github.com/couchbase/indexing/secondary/indexer.(*statsManager).(github.com/couchbase/indexing/secondary/indexer.handleStorageStatsReq)-fm+0x47 goproj/src/github.com/couchbase/indexing/secondary/indexer/stats_manager.go:1120 # 0x72e643 net/http.HandlerFunc.ServeHTTP+0x43 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:1726 # 0x72fa2e net/http.(*ServeMux).ServeHTTP+0x7e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:2022 # 0x73051c net/http.serverHandler.ServeHTTP+0x7c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:2202 # 0x72cec6 net/http.(*conn).serve+0x4b6 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:1579 1 @ 0x43487a 0x43496e 0x40c39f 0x40c065 0x5f1709 0x5f2b1e 0x4659c1 # 0x5f1708 github.com/couchbase/indexing/secondary/indexer.(*statsManager).getStorageStats+0x138 goproj/src/github.com/couchbase/indexing/secondary/indexer/stats_manager.go:1230 # 0x5f2b1d github.com/couchbase/indexing/secondary/indexer.(*statsManager).runStatsDumpLogger+0x39d goproj/src/github.com/couchbase/indexing/secondary/indexer/stats_manager.go:1350 1 @ 0x43487a 0x43496e 0x40c39f 0x40c065 0x64c6b0 0x4659c1 # 0x64c6af github.com/couchbase/indexing/secondary/indexer.(*statsManager).tryUpdateStats.func1+0x10f goproj/src/github.com/couchbase/indexing/secondary/indexer/stats_manager.go:1158 1 @ 0x43487a 0x43496e 0x40c39f 0x40c065 0x65ff84 0x4659c1 # 0x65ff83 github.com/couchbase/indexing/secondary/logging.PeriodicProfile.func1+0x53 goproj/src/github.com/couchbase/indexing/secondary/logging/logging.go:199 1 @ 0x43487a 0x43496e 0x40c39f 0x40c0a5 0x4df96f 0x4659c1 # 0x4df96e github.com/couchbase/indexing/secondary/indexer.(*compactionManager).run+0xae goproj/src/github.com/couchbase/indexing/secondary/indexer/compaction_manager.go:731 1 @ 0x43487a 0x43496e 0x40c39f 0x40c0a5 0x4e04bb 0x4659c1 # 0x4e04ba github.com/couchbase/indexing/secondary/indexer.(*cpuCollector).runCollectStats+0x10a goproj/src/github.com/couchbase/indexing/secondary/indexer/cpu.go:70 1 @ 0x43487a 0x43496e 0x40c39f 0x40c0a5 0x4e1d06 0x4659c1 # 0x4e1d05 github.com/couchbase/indexing/secondary/indexer.(*DDLServiceMgr).run+0x85 goproj/src/github.com/couchbase/indexing/secondary/indexer/ddl_service_manager.go:159 1 @ 0x43487a 0x43496e 0x40c39f 0x40c0a5 0x541403 0x4659c1 # 0x541402 github.com/couchbase/indexing/secondary/indexer.(*kvSender).run+0x62 goproj/src/github.com/couchbase/indexing/secondary/indexer/kv_sender.go:85 1 @ 0x43487a 0x43496e 0x40c39f 0x40c0a5 0x562312 0x4659c1 # 0x562311 github.com/couchbase/indexing/secondary/indexer.(*mutationMgr).run+0xd1 goproj/src/github.com/couchbase/indexing/secondary/indexer/mutation_manager.go:122 1 @ 0x43487a 0x43496e 0x40c39f 0x40c0a5 0x5833b7 0x4659c1 # 0x5833b6 github.com/couchbase/indexing/secondary/indexer.(*ServiceMgr).run+0x66 goproj/src/github.com/couchbase/indexing/secondary/indexer/rebalance_service_manager.go:222 1 @ 0x43487a 0x43496e 0x40c39f 0x40c0a5 0x5b7d33 0x4659c1 # 0x5b7d32 github.com/couchbase/indexing/secondary/indexer.(*scanCoordinator).run+0x62 goproj/src/github.com/couchbase/indexing/secondary/indexer/scan_coordinator.go:122 1 @ 0x43487a 0x43496e 0x40c39f 0x40c0a5 0x5b7ea0 0x4659c1 # 0x5b7e9f github.com/couchbase/indexing/secondary/indexer.(*scanCoordinator).listenSnapshot+0x5f goproj/src/github.com/couchbase/indexing/secondary/indexer/scan_coordinator.go:140 1 @ 0x43487a 0x43496e 0x40c39f 0x40c0a5 0x5e2d03 0x4659c1 # 0x5e2d02 github.com/couchbase/indexing/secondary/indexer.(*settingsManager).run+0x62 goproj/src/github.com/couchbase/indexing/secondary/indexer/settings.go:243 1 @ 0x43487a 0x43496e 0x40c39f 0x40c0a5 0x5f2384 0x4659c1 # 0x5f2383 github.com/couchbase/indexing/secondary/indexer.(*statsManager).run+0x63 goproj/src/github.com/couchbase/indexing/secondary/indexer/stats_manager.go:1309 1 @ 0x43487a 0x43496e 0x40c39f 0x40c0a5 0x5f34f3 0x4659c1 # 0x5f34f2 github.com/couchbase/indexing/secondary/indexer.(*storageMgr).run+0x62 goproj/src/github.com/couchbase/indexing/secondary/indexer/storage_manager.go:153 1 @ 0x43487a 0x43496e 0x40c39f 0x40c0a5 0x5fd765 0x4659c1 # 0x5fd764 github.com/couchbase/indexing/secondary/indexer.(*mutationStreamReader).listenSupvCmd+0x84 goproj/src/github.com/couchbase/indexing/secondary/indexer/stream_reader.go:203 1 @ 0x43487a 0x43496e 0x40c39f 0x40c0a5 0x610643 0x4659c1 # 0x610642 github.com/couchbase/indexing/secondary/indexer.(*timekeeper).run+0x62 goproj/src/github.com/couchbase/indexing/secondary/indexer/timekeeper.go:112 1 @ 0x43487a 0x43496e 0x444aed 0x444780 0x6bd8f0 0xc97fa3 0xc9b756 0xc98a46 0x4659c1 # 0x44477f sync.runtime_Semacquire+0x2f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/sema.go:47 # 0x6bd8ef sync.(*Mutex).Lock+0xcf /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/sync/mutex.go:85 # 0xc97fa2 github.com/couchbase/gometa/common.(*PeerPipe).Close+0x52 goproj/src/github.com/couchbase/gometa/common/peerPipe.go:91 # 0xc9b755 github.com/couchbase/gometa/common.(*PeerPipe).doSend.func1+0x55 goproj/src/github.com/couchbase/gometa/common/peerPipe.go:151 # 0xc98a45 github.com/couchbase/gometa/common.(*PeerPipe).doSend+0x625 goproj/src/github.com/couchbase/gometa/common/peerPipe.go:181 1 @ 0x43487a 0x43496e 0x444aed 0x444780 0x6bd8f0 0xc97fa3 0xc9b956 0xc997eb 0x4659c1 # 0x44477f sync.runtime_Semacquire+0x2f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/sema.go:47 # 0x6bd8ef sync.(*Mutex).Lock+0xcf /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/sync/mutex.go:85 # 0xc97fa2 github.com/couchbase/gometa/common.(*PeerPipe).Close+0x52 goproj/src/github.com/couchbase/gometa/common/peerPipe.go:91 # 0xc9b955 github.com/couchbase/gometa/common.(*PeerPipe).doReceive.func1+0x55 goproj/src/github.com/couchbase/gometa/common/peerPipe.go:198 # 0xc997ea github.com/couchbase/gometa/common.(*PeerPipe).doReceive+0x9da goproj/src/github.com/couchbase/gometa/common/peerPipe.go:208 1 @ 0x43487a 0x43496e 0x445182 0x6bd640 0x9da8a3 0x9da0bf 0x9d8666 0x9bed35 0x4e945e 0x4e42d3 0x4e623e 0x4659c1 # 0x445181 sync.runtime_notifyListWait+0x121 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/sema.go:267 # 0x6bd63f sync.(*Cond).Wait+0x7f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/sync/cond.go:57 # 0x9da8a2 github.com/couchbase/indexing/secondary/manager/client.(*watcher).makeRequest+0x322 goproj/src/github.com/couchbase/indexing/secondary/manager/client/metadata_provider.go:3897 # 0x9da0be github.com/couchbase/indexing/secondary/manager/client.(*watcher).refreshServiceMap+0xae goproj/src/github.com/couchbase/indexing/secondary/manager/client/metadata_provider.go:3825 # 0x9d8665 github.com/couchbase/indexing/secondary/manager/client.(*watcher).notifyReady+0x75 goproj/src/github.com/couchbase/indexing/secondary/manager/client/metadata_provider.go:3550 # 0x9bed34 github.com/couchbase/indexing/secondary/manager/client.(*MetadataProvider).WatchMetadata+0x5e4 goproj/src/github.com/couchbase/indexing/secondary/manager/client/metadata_provider.go:271 # 0x4e945d github.com/couchbase/indexing/secondary/indexer.(*DDLServiceMgr).newMetadataProvider+0x5cd goproj/src/github.com/couchbase/indexing/secondary/indexer/ddl_service_manager.go:1180 # 0x4e42d2 github.com/couchbase/indexing/secondary/indexer.(*DDLServiceMgr).handleCreateCommand+0xe2 goproj/src/github.com/couchbase/indexing/secondary/indexer/ddl_service_manager.go:542 # 0x4e623d github.com/couchbase/indexing/secondary/indexer.(*DDLServiceMgr).processCreateCommand+0x13d goproj/src/github.com/couchbase/indexing/secondary/indexer/ddl_service_manager.go:761 1 @ 0x43487a 0x43496e 0x445182 0x6bd640 0xcc4cf4 0x99cc90 0x99d3d5 0x99adff 0x994357 0x993fbb 0x4659c1 # 0x445181 sync.runtime_notifyListWait+0x121 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/sema.go:267 # 0x6bd63f sync.(*Cond).Wait+0x7f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/sync/cond.go:57 # 0xcc4cf3 github.com/couchbase/gometa/server.(*EmbeddedServer).Broadcast+0x333 goproj/src/github.com/couchbase/gometa/server/embeddedServer.go:193 # 0x99cc8f github.com/couchbase/indexing/secondary/manager.(*LocalRepoRef).broadcast+0x6f goproj/src/github.com/couchbase/indexing/secondary/manager/meta_repo.go:677 # 0x99d3d4 github.com/couchbase/indexing/secondary/manager.(*MetadataRepo).broadcast+0x64 goproj/src/github.com/couchbase/indexing/secondary/manager/meta_repo.go:912 # 0x99adfe github.com/couchbase/indexing/secondary/manager.(*MetadataRepo).BroadcastServiceMap+0x7e goproj/src/github.com/couchbase/indexing/secondary/manager/meta_repo.go:403 # 0x994356 github.com/couchbase/indexing/secondary/manager.(*updator).checkServiceMap+0x2b6 goproj/src/github.com/couchbase/indexing/secondary/manager/lifecycle.go:3157 # 0x993fba github.com/couchbase/indexing/secondary/manager.(*updator).run+0x1ba goproj/src/github.com/couchbase/indexing/secondary/manager/lifecycle.go:3122 1 @ 0x43487a 0x43496e 0x445182 0x6bd640 0xcc4cf4 0x99cc90 0x99d3d5 0x99aedf 0x987325 0x97d673 0x97c4d4 0x4659c1 # 0x445181 sync.runtime_notifyListWait+0x121 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/sema.go:267 # 0x6bd63f sync.(*Cond).Wait+0x7f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/sync/cond.go:57 # 0xcc4cf3 github.com/couchbase/gometa/server.(*EmbeddedServer).Broadcast+0x333 goproj/src/github.com/couchbase/gometa/server/embeddedServer.go:193 # 0x99cc8f github.com/couchbase/indexing/secondary/manager.(*LocalRepoRef).broadcast+0x6f goproj/src/github.com/couchbase/indexing/secondary/manager/meta_repo.go:677 # 0x99d3d4 github.com/couchbase/indexing/secondary/manager.(*MetadataRepo).broadcast+0x64 goproj/src/github.com/couchbase/indexing/secondary/manager/meta_repo.go:912 # 0x99aede github.com/couchbase/indexing/secondary/manager.(*MetadataRepo).BroadcastIndexStats+0x7e goproj/src/github.com/couchbase/indexing/secondary/manager/meta_repo.go:418 # 0x987324 github.com/couchbase/indexing/secondary/manager.(*LifecycleMgr).handleBroadcastStats+0x2b4 goproj/src/github.com/couchbase/indexing/secondary/manager/lifecycle.go:1554 # 0x97d672 github.com/couchbase/indexing/secondary/manager.(*LifecycleMgr).dispatchRequest+0x1032 goproj/src/github.com/couchbase/indexing/secondary/manager/lifecycle.go:347 # 0x97c4d3 github.com/couchbase/indexing/secondary/manager.(*LifecycleMgr).processRequest+0x393 goproj/src/github.com/couchbase/indexing/secondary/manager/lifecycle.go:291 1 @ 0x43487a 0x43496e 0x44f9d1 0x4896a1 0x4659c1 # 0x44f9d0 time.Sleep+0xe0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/time.go:59 # 0x4896a0 github.com/couchbase/indexing/secondary/common.pollForDeletedBuckets+0x40 goproj/src/github.com/couchbase/indexing/secondary/common/dcp_seqno.go:341 1 @ 0x43487a 0x43496e 0x44f9d1 0x53d103 0x4659c1 # 0x44f9d0 time.Sleep+0xe0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/time.go:59 # 0x53d102 github.com/couchbase/indexing/secondary/indexer.(*indexer).monitorMemUsage+0x282 goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:6306 1 @ 0x43487a 0x43496e 0x44f9d1 0x53fad2 0x4659c1 # 0x44f9d0 time.Sleep+0xe0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/time.go:59 # 0x53fad1 github.com/couchbase/indexing/secondary/indexer.(*indexer).logMemstats+0x191 goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:6586 1 @ 0x43487a 0x43496e 0x44f9d1 0x58b516 0x4659c1 # 0x44f9d0 time.Sleep+0xe0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/time.go:59 # 0x58b515 github.com/couchbase/indexing/secondary/indexer.(*ServiceMgr).rebalanceJanitor+0x45 goproj/src/github.com/couchbase/indexing/secondary/indexer/rebalance_service_manager.go:1204 1 @ 0x43487a 0x43496e 0x44f9d1 0x99048f 0x4659c1 # 0x44f9d0 time.Sleep+0xe0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/time.go:59 # 0x99048e github.com/couchbase/indexing/secondary/manager.(*builder).run+0x30e goproj/src/github.com/couchbase/indexing/secondary/manager/lifecycle.go:2759 1 @ 0x43487a 0x43496e 0x44f9d1 0xa9f9da 0x4659c1 # 0x44f9d0 time.Sleep+0xe0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/time.go:59 # 0xa9f9d9 github.com/couchbase/plasma.RunMemQuotaTuner+0x259 goproj/src/github.com/couchbase/plasma/mem.go:129 1 @ 0x43487a 0x443f15 0x442cac 0x48443a 0x4659c1 # 0x484439 github.com/couchbase/indexing/secondary/common.(*ClusterInfoClient).watchClusterChanges+0x429 goproj/src/github.com/couchbase/indexing/secondary/common/cluster_info.go:684 1 @ 0x43487a 0x443f15 0x442cac 0x4da18d 0x4659c1 # 0x4da18c github.com/couchbase/indexing/secondary/indexer.(*compactionDaemon).loop+0x2dc goproj/src/github.com/couchbase/indexing/secondary/indexer/compaction_manager.go:117 1 @ 0x43487a 0x443f15 0x442cac 0x50a1c8 0x4659c1 # 0x50a1c7 github.com/couchbase/indexing/secondary/indexer.(*indexer).listenAdminMsgs+0x2c7 goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:602 1 @ 0x43487a 0x443f15 0x442cac 0x50a7d4 0x4659c1 # 0x50a7d3 github.com/couchbase/indexing/secondary/indexer.(*indexer).listenWorkerMsgs+0x2c3 goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:661 1 @ 0x43487a 0x443f15 0x442cac 0x562592 0x4659c1 # 0x562591 github.com/couchbase/indexing/secondary/indexer.(*mutationMgr).handleWorkerMsgs+0x161 goproj/src/github.com/couchbase/indexing/secondary/indexer/mutation_manager.go:146 1 @ 0x43487a 0x443f15 0x442cac 0x562a37 0x4659c1 # 0x562a36 github.com/couchbase/indexing/secondary/indexer.(*mutationMgr).listenWorkerMsgs+0x496 goproj/src/github.com/couchbase/indexing/secondary/indexer/mutation_manager.go:169 1 @ 0x43487a 0x443f15 0x442cac 0x58f0fc 0x583c14 0x9bb35e 0x9ba6e9 0x9bab14 0x9bbd00 0x462f0c 0x681298 0x680bb4 0x77abe8 0x4659c1 # 0x58f0fb github.com/couchbase/indexing/secondary/indexer.(*ServiceMgr).wait+0x44b goproj/src/github.com/couchbase/indexing/secondary/indexer/rebalance_service_manager.go:1630 # 0x583c13 github.com/couchbase/indexing/secondary/indexer.(*ServiceMgr).GetTaskList+0x133 goproj/src/github.com/couchbase/indexing/secondary/indexer/rebalance_service_manager.go:305 # 0x9bb35d github.com/couchbase/cbauth/service.serviceAPI.GetTaskList.func1+0x7d godeps/src/github.com/couchbase/cbauth/service/revrpc.go:99 # 0x9ba6e8 github.com/couchbase/cbauth/service.withTimeout+0x88 godeps/src/github.com/couchbase/cbauth/service/revrpc.go:46 # 0x9bab13 github.com/couchbase/cbauth/service.serviceAPI.GetTaskList+0xe3 godeps/src/github.com/couchbase/cbauth/service/revrpc.go:103 # 0x9bbcff github.com/couchbase/cbauth/service.(*serviceAPI).GetTaskList+0x8f :11 # 0x462f0b runtime.call64+0x4b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/asm_amd64.s:480 # 0x681297 reflect.Value.call+0x5c7 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/reflect/value.go:434 # 0x680bb3 reflect.Value.Call+0xa3 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/reflect/value.go:302 # 0x77abe7 net/rpc.(*service).call+0x147 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/rpc/server.go:383 1 @ 0x43487a 0x443f15 0x442cac 0x58f0fc 0x5843a7 0x9bb13e 0x9ba6e9 0x9ba9a4 0x9bbbf0 0x462f0c 0x681298 0x680bb4 0x77abe8 0x4659c1 # 0x58f0fb github.com/couchbase/indexing/secondary/indexer.(*ServiceMgr).wait+0x44b goproj/src/github.com/couchbase/indexing/secondary/indexer/rebalance_service_manager.go:1630 # 0x5843a6 github.com/couchbase/indexing/secondary/indexer.(*ServiceMgr).GetCurrentTopology+0x136 goproj/src/github.com/couchbase/indexing/secondary/indexer/rebalance_service_manager.go:355 # 0x9bb13d github.com/couchbase/cbauth/service.serviceAPI.GetCurrentTopology.func1+0x7d godeps/src/github.com/couchbase/cbauth/service/revrpc.go:76 # 0x9ba6e8 github.com/couchbase/cbauth/service.withTimeout+0x88 godeps/src/github.com/couchbase/cbauth/service/revrpc.go:46 # 0x9ba9a3 github.com/couchbase/cbauth/service.serviceAPI.GetCurrentTopology+0xe3 godeps/src/github.com/couchbase/cbauth/service/revrpc.go:80 # 0x9bbbef github.com/couchbase/cbauth/service.(*serviceAPI).GetCurrentTopology+0x8f :10 # 0x462f0b runtime.call64+0x4b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/asm_amd64.s:480 # 0x681297 reflect.Value.call+0x5c7 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/reflect/value.go:434 # 0x680bb3 reflect.Value.Call+0xa3 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/reflect/value.go:302 # 0x77abe7 net/rpc.(*service).call+0x147 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/rpc/server.go:383 1 @ 0x43487a 0x443f15 0x442cac 0x5fd6be 0x4659c1 # 0x5fd6bd github.com/couchbase/indexing/secondary/indexer.(*mutationStreamReader).run+0x32d goproj/src/github.com/couchbase/indexing/secondary/indexer/stream_reader.go:162 1 @ 0x43487a 0x443f15 0x442cac 0x5fe935 0x4659c1 # 0x5fe934 github.com/couchbase/indexing/secondary/indexer.(*mutationStreamReader).syncWorker+0x1a4 goproj/src/github.com/couchbase/indexing/secondary/indexer/stream_reader.go:355 1 @ 0x43487a 0x443f15 0x442cac 0x75ab62 0x4659c1 # 0x75ab61 github.com/couchbase/cbauth/cbauthimpl.(*certNotifier).loop+0x141 godeps/src/github.com/couchbase/cbauth/cbauthimpl/impl.go:214 1 @ 0x43487a 0x443f15 0x442cac 0x940659 0x94003c 0x940a85 0x5930d4 0x4659c1 # 0x940658 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x5d8 godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0x94003b github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x5b godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0x940a84 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x5930d3 github.com/couchbase/indexing/secondary/indexer.(*ServiceMgr).listenMoveIndex+0xf3 goproj/src/github.com/couchbase/indexing/secondary/indexer/rebalance_service_manager.go:2116 1 @ 0x43487a 0x443f15 0x442cac 0x940659 0x94003c 0x940a85 0x646455 0x4946d2 0x64661d 0x4659c1 # 0x940658 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x5d8 godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0x94003b github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x5b godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0x940a84 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x646454 github.com/couchbase/indexing/secondary/indexer.NewSettingsManager.func1.1+0x94 goproj/src/github.com/couchbase/indexing/secondary/indexer/settings.go:82 # 0x4946d1 github.com/couchbase/indexing/secondary/common.(*RetryHelper).Run+0x51 goproj/src/github.com/couchbase/indexing/secondary/common/retry_helper.go:36 # 0x64661c github.com/couchbase/indexing/secondary/indexer.NewSettingsManager.func1+0xbc goproj/src/github.com/couchbase/indexing/secondary/indexer/settings.go:86 1 @ 0x43487a 0x443f15 0x442cac 0x990154 0x4659c1 # 0x990153 github.com/couchbase/indexing/secondary/manager.(*janitor).run+0x4d3 goproj/src/github.com/couchbase/indexing/secondary/manager/lifecycle.go:2698 1 @ 0x43487a 0x443f15 0x442cac 0x998515 0x4659c1 # 0x998514 github.com/couchbase/indexing/secondary/manager.(*IndexManager).monitorBucket+0x2c4 goproj/src/github.com/couchbase/indexing/secondary/manager/manager.go:666 1 @ 0x43487a 0x443f15 0x442cac 0xbce704 0x4659c1 # 0xbce703 github.com/couchbase/indexing/secondary/dataport.(*Server).genServer+0x963 goproj/src/github.com/couchbase/indexing/secondary/dataport/server.go:338 1 @ 0x43487a 0x443f15 0x442cac 0xcb6ff6 0x4659c1 # 0xcb6ff5 github.com/couchbase/gometa/protocol.(*LeaderServer).listenFollower+0x4b5 goproj/src/github.com/couchbase/gometa/protocol/leaderServer.go:186 1 @ 0x4659c1 1 @ 0x6bd485 0x6ba305 0x6b8aa9 0x46f61f 0x46ce89 0x49b545 0x4659c1 # 0x6bd484 syscall.Syscall+0x4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/syscall/asm_linux_amd64.s:18 # 0x6ba304 syscall.read+0x54 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/syscall/zsyscall_linux_amd64.go:783 # 0x6b8aa8 syscall.Read+0x48 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/syscall/syscall_unix.go:161 # 0x46f61e os.(*File).read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/os/file_unix.go:228 # 0x46ce88 os.(*File).Read+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/os/file.go:101 # 0x49b544 github.com/couchbase/indexing/secondary/common.ExitOnStdinClose+0x54 goproj/src/github.com/couchbase/indexing/secondary/common/util.go:355 1 @ 0x6bd485 0x6ba305 0x6b8aa9 0x7a6bdf 0x7b7d00 0xc99a02 0xc98ea4 0x4659c1 # 0x6bd484 syscall.Syscall+0x4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/syscall/asm_linux_amd64.s:18 # 0x6ba304 syscall.read+0x54 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/syscall/zsyscall_linux_amd64.go:783 # 0x6b8aa8 syscall.Read+0x48 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/syscall/syscall_unix.go:161 # 0x7a6bde net.(*netFD).Read+0x10e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:239 # 0x7b7cff net.(*conn).Read+0x6f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/net.go:173 # 0xc99a01 github.com/couchbase/gometa/common.(*PeerPipe).readBytes+0xc1 goproj/src/github.com/couchbase/gometa/common/peerPipe.go:270 # 0xc98ea3 github.com/couchbase/gometa/common.(*PeerPipe).doReceive+0x93 goproj/src/github.com/couchbase/gometa/common/peerPipe.go:203 1 @ 0xa188af 0xa186b0 0xa15361 0xd4a106 0xd4a485 0x72e644 0x72fa2f 0x73051d 0x72cec7 0x4659c1 # 0xa188ae runtime/pprof.writeRuntimeProfile+0x9e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/pprof/pprof.go:614 # 0xa186af runtime/pprof.writeGoroutine+0x9f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/pprof/pprof.go:576 # 0xa15360 runtime/pprof.(*Profile).WriteTo+0x340 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/pprof/pprof.go:298 # 0xd4a105 net/http/pprof.handler.ServeHTTP+0x1a5 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/pprof/pprof.go:209 # 0xd4a484 net/http/pprof.Index+0x204 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/pprof/pprof.go:221 # 0x72e643 net/http.HandlerFunc.ServeHTTP+0x43 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:1726 # 0x72fa2e net/http.(*ServeMux).ServeHTTP+0x7e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:2022 # 0x73051c net/http.serverHandler.ServeHTTP+0x7c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:2202 # 0x72cec6 net/http.(*conn).serve+0x4b6 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:1579