============================================================================== Indexer Go routine dump: ['curl', '-sS', '--proxy', '', '-K-', 'http://127.0.0.1:9102/debug/pprof/goroutine?debug=1'] ============================================================================== goroutine profile: total 741 180 @ 0x43488a 0x43497e 0x40c3af 0x40c0b5 0xa20067 0x4659d1 # 0xa20066 github.com/couchbase/indexing/secondary/memdb.(*MemDB).freeWorker+0x56 goproj/src/github.com/couchbase/indexing/secondary/memdb/memdb.go:641 180 @ 0x43488a 0x443f25 0x442cbc 0x5513b5 0x4659d1 # 0x5513b4 github.com/couchbase/indexing/secondary/indexer.(*memdbSlice).handleCommandsWorker+0x664 goproj/src/github.com/couchbase/indexing/secondary/indexer/memdb_slice_impl.go:375 180 @ 0x43488a 0x443f25 0x442cbc 0xa1ffce 0x4659d1 # 0xa1ffcd github.com/couchbase/indexing/secondary/memdb.(*MemDB).collectionWorker+0x40d goproj/src/github.com/couchbase/indexing/secondary/memdb/memdb.go:619 32 @ 0x43488a 0x443f25 0x442cbc 0x6002f0 0x4659d1 # 0x6002ef github.com/couchbase/indexing/secondary/indexer.(*streamWorker).start+0x23f goproj/src/github.com/couchbase/indexing/secondary/indexer/stream_reader.go:542 14 @ 0x43488a 0x43497e 0x40b6b8 0x40b47d 0x639572 0xa26514 0xa25d54 0x4659d1 # 0x639571 github.com/couchbase/indexing/secondary/indexer.(*memdbSlice).doPersistSnapshot.func1+0x61 goproj/src/github.com/couchbase/indexing/secondary/indexer/memdb_slice_impl.go:767 # 0xa26513 github.com/couchbase/indexing/secondary/memdb.(*MemDB).StoreToDisk.func5+0xf3 goproj/src/github.com/couchbase/indexing/secondary/memdb/memdb.go:963 # 0xa25d53 github.com/couchbase/indexing/secondary/memdb.(*MemDB).Visitor.func2+0x253 goproj/src/github.com/couchbase/indexing/secondary/memdb/memdb.go:777 13 @ 0x43488a 0x443f25 0x442cbc 0x7412bd 0x4659d1 # 0x7412bc net/http.(*persistConn).writeLoop+0x3bc /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transport.go:1646 12 @ 0x43488a 0x43497e 0x444afd 0x444790 0x6bee07 0xa20d1f 0xa21a10 0x554f22 0x4659d1 # 0x44478f sync.runtime_Semacquire+0x2f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/sema.go:47 # 0x6bee06 sync.(*WaitGroup).Wait+0x96 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/sync/waitgroup.go:131 # 0xa20d1e github.com/couchbase/indexing/secondary/memdb.(*MemDB).Visitor+0x2ae goproj/src/github.com/couchbase/indexing/secondary/memdb/memdb.go:792 # 0xa21a0f github.com/couchbase/indexing/secondary/memdb.(*MemDB).StoreToDisk+0x82f goproj/src/github.com/couchbase/indexing/secondary/memdb/memdb.go:971 # 0x554f21 github.com/couchbase/indexing/secondary/indexer.(*memdbSlice).doPersistSnapshot+0x471 goproj/src/github.com/couchbase/indexing/secondary/indexer/memdb_slice_impl.go:776 8 @ 0x43488a 0x42f14b 0x42e8a9 0x7a53d8 0x7a5444 0x7a6c81 0x7b7d10 0xcd9a19 0xcda4fa 0xcd93b0 0xbc831c 0x4659d1 # 0x42e8a8 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53d7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5443 net.(*pollDesc).waitRead+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:78 # 0x7a6c80 net.(*netFD).Read+0x1a0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:243 # 0x7b7d0f net.(*conn).Read+0x6f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/net.go:173 # 0xcd9a18 github.com/couchbase/indexing/secondary/transport.fullRead+0x88 goproj/src/github.com/couchbase/indexing/secondary/transport/transport.go:189 # 0xcda4f9 github.com/couchbase/indexing/secondary/transport.Receive+0x89 goproj/src/github.com/couchbase/indexing/secondary/transport/util.go:60 # 0xcd93af github.com/couchbase/indexing/secondary/transport.(*TransportPacket).Receive+0x6f goproj/src/github.com/couchbase/indexing/secondary/transport/transport.go:119 # 0xbc831b github.com/couchbase/indexing/secondary/queryport.(*Server).doReceive+0x3bb goproj/src/github.com/couchbase/indexing/secondary/queryport/server.go:225 8 @ 0x43488a 0x43497e 0x40c3af 0x40c0b5 0x487a0c 0x4659d1 # 0x487a0b github.com/couchbase/indexing/secondary/common.(*vbSeqnosReader).Routine+0x7b goproj/src/github.com/couchbase/indexing/secondary/common/dcp_seqno.go:104 8 @ 0x43488a 0x43497e 0x40c3af 0x40c0b5 0xbc7d83 0x4659d1 # 0xbc7d82 github.com/couchbase/indexing/secondary/queryport.(*Server).handleConnection+0x262 goproj/src/github.com/couchbase/indexing/secondary/queryport/server.go:197 8 @ 0x43488a 0x443f25 0x442cbc 0x6525ac 0x4659d1 # 0x6525ab github.com/couchbase/indexing/secondary/indexer.(*timekeeper).startTimer.func1+0x13b goproj/src/github.com/couchbase/indexing/secondary/indexer/timekeeper.go:2947 8 @ 0x43488a 0x443f25 0x442cbc 0xbc8915 0x4659d1 # 0xbc8914 github.com/couchbase/indexing/secondary/queryport.(*Server).doPing+0x1b4 goproj/src/github.com/couchbase/indexing/secondary/queryport/server.go:257 7 @ 0x43488a 0x443f25 0x442cbc 0x73ff39 0x4659d1 # 0x73ff38 net/http.(*persistConn).readLoop+0x9c8 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transport.go:1541 6 @ 0x43488a 0x42f14b 0x42e8a9 0x7a53d8 0x7a5444 0x7a6c81 0x7b7d10 0x73eaf4 0xc21efc 0xc220e2 0x73f711 0x4659d1 # 0x42e8a8 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53d7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5443 net.(*pollDesc).waitRead+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:78 # 0x7a6c80 net.(*netFD).Read+0x1a0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:243 # 0x7b7d0f net.(*conn).Read+0x6f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/net.go:173 # 0x73eaf3 net/http.(*persistConn).Read+0x153 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transport.go:1261 # 0xc21efb bufio.(*Reader).fill+0x10b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:97 # 0xc220e1 bufio.(*Reader).Peek+0x61 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:129 # 0x73f710 net/http.(*persistConn).readLoop+0x1a0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transport.go:1418 5 @ 0x43488a 0x42f14b 0x42e8a9 0x7a53d8 0x7a5444 0x7a6c81 0x7b7d10 0x73eaf4 0xc21efc 0xc22ba5 0xc33aa4 0xc334e2 0xc3374b 0x736771 0x7366b6 0x742756 0x773a3a 0x773688 0x7731ce 0x94327d 0x4659d1 # 0x42e8a8 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53d7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5443 net.(*pollDesc).waitRead+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:78 # 0x7a6c80 net.(*netFD).Read+0x1a0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:243 # 0x7b7d0f net.(*conn).Read+0x6f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/net.go:173 # 0x73eaf3 net/http.(*persistConn).Read+0x153 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transport.go:1261 # 0xc21efb bufio.(*Reader).fill+0x10b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:97 # 0xc22ba4 bufio.(*Reader).ReadSlice+0xb4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:330 # 0xc33aa3 net/http/internal.readChunkLine+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/internal/chunked.go:110 # 0xc334e1 net/http/internal.(*chunkedReader).beginChunk+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/internal/chunked.go:47 # 0xc3374a net/http/internal.(*chunkedReader).Read+0x7a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/internal/chunked.go:77 # 0x736770 net/http.(*body).readLocked+0x60 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transfer.go:651 # 0x7366b5 net/http.(*body).Read+0xf5 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transfer.go:643 # 0x742755 net/http.(*bodyEOFSignal).Read+0x115 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transport.go:1974 # 0x773a39 encoding/json.(*Decoder).refill+0xf9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/encoding/json/stream.go:152 # 0x773687 encoding/json.(*Decoder).readValue+0x197 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/encoding/json/stream.go:128 # 0x7731cd encoding/json.(*Decoder).Decode+0x8d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/encoding/json/stream.go:57 # 0x94327c github.com/couchbase/cbauth/metakv.doRunObserveChildren.func1+0x11c godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:270 3 @ 0x43488a 0x42f14b 0x42e8a9 0x7a53d8 0x7a5444 0x7a6c81 0x7b7d10 0x727a44 0xc21efc 0xc22ba5 0xc22d27 0xc1e6ce 0xc1e4cf 0x7224b5 0x728dbd 0x72cdf3 0x4659d1 # 0x42e8a8 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53d7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5443 net.(*pollDesc).waitRead+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:78 # 0x7a6c80 net.(*netFD).Read+0x1a0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:243 # 0x7b7d0f net.(*conn).Read+0x6f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/net.go:173 # 0x727a43 net/http.(*connReader).Read+0x143 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:586 # 0xc21efb bufio.(*Reader).fill+0x10b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:97 # 0xc22ba4 bufio.(*Reader).ReadSlice+0xb4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:330 # 0xc22d26 bufio.(*Reader).ReadLine+0x36 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:359 # 0xc1e6cd net/textproto.(*Reader).readLineSlice+0x5d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/textproto/reader.go:55 # 0xc1e4ce net/textproto.(*Reader).ReadLine+0x2e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/textproto/reader.go:36 # 0x7224b4 net/http.readRequest+0xa4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/request.go:793 # 0x728dbc net/http.(*conn).readRequest+0x10c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:765 # 0x72cdf2 net/http.(*conn).serve+0x3d2 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:1532 3 @ 0x43488a 0x443f25 0x442cbc 0x940669 0x94004c 0x940a95 0x9e3a71 0x4946e2 0x9e3c84 0x4659d1 # 0x940668 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x5d8 godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0x94004b github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x5b godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0x940a94 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x9e3a70 github.com/couchbase/indexing/secondary/manager/common.(*CommandListener).ListenTokens.func2.1+0x70 goproj/src/github.com/couchbase/indexing/secondary/manager/common/token.go:658 # 0x4946e1 github.com/couchbase/indexing/secondary/common.(*RetryHelper).Run+0x51 goproj/src/github.com/couchbase/indexing/secondary/common/retry_helper.go:36 # 0x9e3c83 github.com/couchbase/indexing/secondary/manager/common.(*CommandListener).ListenTokens.func2+0x103 goproj/src/github.com/couchbase/indexing/secondary/manager/common/token.go:663 2 @ 0x43488a 0x42f14b 0x42e8a9 0x7a53d8 0x7a5444 0x7a6c81 0x7b7d10 0xc99a12 0xc98eb4 0x4659d1 # 0x42e8a8 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53d7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5443 net.(*pollDesc).waitRead+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:78 # 0x7a6c80 net.(*netFD).Read+0x1a0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:243 # 0x7b7d0f net.(*conn).Read+0x6f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/net.go:173 # 0xc99a11 github.com/couchbase/gometa/common.(*PeerPipe).readBytes+0xc1 goproj/src/github.com/couchbase/gometa/common/peerPipe.go:270 # 0xc98eb3 github.com/couchbase/gometa/common.(*PeerPipe).doReceive+0x93 goproj/src/github.com/couchbase/gometa/common/peerPipe.go:203 2 @ 0x43488a 0x43497e 0x40c3af 0x40c0b5 0xc984d5 0x4659d1 # 0xc984d4 github.com/couchbase/gometa/common.(*PeerPipe).doSend+0xa4 goproj/src/github.com/couchbase/gometa/common/peerPipe.go:155 2 @ 0x43488a 0x443f25 0x442cbc 0x9bb07f 0x4659d1 # 0x9bb07e github.com/couchbase/cbauth/service.withTimeout.func1+0x13e godeps/src/github.com/couchbase/cbauth/service/revrpc.go:38 2 @ 0x43488a 0x443f25 0x442cbc 0xcb2c33 0x4659d1 # 0xcb2c32 github.com/couchbase/gometa/protocol.(*messageListener).start+0x4d2 goproj/src/github.com/couchbase/gometa/protocol/leader.go:403 1 @ 0x415f42 0x447d07 0xd4a982 0x4659d1 # 0x447d06 os/signal.signal_recv+0x156 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/sigqueue.go:116 # 0xd4a981 os/signal.loop+0x21 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/os/signal/signal_unix.go:22 1 @ 0x43488a 0x42f14b 0x42e8a9 0x7a53d8 0x7a5444 0x7a6c81 0x7b7d10 0x73eaf4 0xc21efc 0xc22ba5 0xc33aa4 0xc334e2 0xc3374b 0x736771 0x7366b6 0x742756 0xc21efc 0xc22ba5 0xc23036 0x82f1a1 0x82ed99 0x494c97 0x4659d1 # 0x42e8a8 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53d7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5443 net.(*pollDesc).waitRead+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:78 # 0x7a6c80 net.(*netFD).Read+0x1a0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:243 # 0x7b7d0f net.(*conn).Read+0x6f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/net.go:173 # 0x73eaf3 net/http.(*persistConn).Read+0x153 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transport.go:1261 # 0xc21efb bufio.(*Reader).fill+0x10b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:97 # 0xc22ba4 bufio.(*Reader).ReadSlice+0xb4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:330 # 0xc33aa3 net/http/internal.readChunkLine+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/internal/chunked.go:110 # 0xc334e1 net/http/internal.(*chunkedReader).beginChunk+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/internal/chunked.go:47 # 0xc3374a net/http/internal.(*chunkedReader).Read+0x7a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/internal/chunked.go:77 # 0x736770 net/http.(*body).readLocked+0x60 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transfer.go:651 # 0x7366b5 net/http.(*body).Read+0xf5 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transfer.go:643 # 0x742755 net/http.(*bodyEOFSignal).Read+0x115 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transport.go:1974 # 0xc21efb bufio.(*Reader).fill+0x10b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:97 # 0xc22ba4 bufio.(*Reader).ReadSlice+0xb4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:330 # 0xc23035 bufio.(*Reader).ReadBytes+0x65 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:408 # 0x82f1a0 github.com/couchbase/indexing/secondary/dcp.(*Client).runObserveStreamingEndpoint+0x2f0 goproj/src/github.com/couchbase/indexing/secondary/dcp/pools.go:379 # 0x82ed98 github.com/couchbase/indexing/secondary/dcp.(*Client).RunObservePool+0x98 goproj/src/github.com/couchbase/indexing/secondary/dcp/pools.go:318 # 0x494c96 github.com/couchbase/indexing/secondary/common.(*serviceNotifierInstance).RunPoolObserver+0x96 goproj/src/github.com/couchbase/indexing/secondary/common/services_notifier.go:79 1 @ 0x43488a 0x42f14b 0x42e8a9 0x7a53d8 0x7a5444 0x7a6c81 0x7b7d10 0x73eaf4 0xc21efc 0xc22ba5 0xc33aa4 0xc334e2 0xc3374b 0x736771 0x7366b6 0x742756 0xc21efc 0xc22ba5 0xc23036 0x82f1a1 0x82ee7e 0x494e97 0x4659d1 # 0x42e8a8 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53d7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5443 net.(*pollDesc).waitRead+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:78 # 0x7a6c80 net.(*netFD).Read+0x1a0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:243 # 0x7b7d0f net.(*conn).Read+0x6f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/net.go:173 # 0x73eaf3 net/http.(*persistConn).Read+0x153 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transport.go:1261 # 0xc21efb bufio.(*Reader).fill+0x10b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:97 # 0xc22ba4 bufio.(*Reader).ReadSlice+0xb4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:330 # 0xc33aa3 net/http/internal.readChunkLine+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/internal/chunked.go:110 # 0xc334e1 net/http/internal.(*chunkedReader).beginChunk+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/internal/chunked.go:47 # 0xc3374a net/http/internal.(*chunkedReader).Read+0x7a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/internal/chunked.go:77 # 0x736770 net/http.(*body).readLocked+0x60 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transfer.go:651 # 0x7366b5 net/http.(*body).Read+0xf5 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transfer.go:643 # 0x742755 net/http.(*bodyEOFSignal).Read+0x115 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/transport.go:1974 # 0xc21efb bufio.(*Reader).fill+0x10b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:97 # 0xc22ba4 bufio.(*Reader).ReadSlice+0xb4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:330 # 0xc23035 bufio.(*Reader).ReadBytes+0x65 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:408 # 0x82f1a0 github.com/couchbase/indexing/secondary/dcp.(*Client).runObserveStreamingEndpoint+0x2f0 goproj/src/github.com/couchbase/indexing/secondary/dcp/pools.go:379 # 0x82ee7d github.com/couchbase/indexing/secondary/dcp.(*Client).RunObserveNodeServices+0xad goproj/src/github.com/couchbase/indexing/secondary/dcp/pools.go:331 # 0x494e96 github.com/couchbase/indexing/secondary/common.(*serviceNotifierInstance).RunServicesObserver+0x96 goproj/src/github.com/couchbase/indexing/secondary/common/services_notifier.go:88 1 @ 0x43488a 0x42f14b 0x42e8a9 0x7a53d8 0x7a5444 0x7a6c81 0x7b7d10 0xc21efc 0xc2255c 0x78862d 0x773a3a 0x773688 0x7731ce 0xc4a5f4 0x77c97c 0x77c44a 0x77ba1a 0x788dfd 0x78970c 0x47b711 0x47cea5 0x4659d1 # 0x42e8a8 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53d7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5443 net.(*pollDesc).waitRead+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:78 # 0x7a6c80 net.(*netFD).Read+0x1a0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:243 # 0x7b7d0f net.(*conn).Read+0x6f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/net.go:173 # 0xc21efb bufio.(*Reader).fill+0x10b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:97 # 0xc2255b bufio.(*Reader).Read+0x1bb /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:209 # 0x78862c github.com/couchbase/cbauth/revrpc.(*minirwc).Read+0x4c godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:101 # 0x773a39 encoding/json.(*Decoder).refill+0xf9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/encoding/json/stream.go:152 # 0x773687 encoding/json.(*Decoder).readValue+0x197 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/encoding/json/stream.go:128 # 0x7731cd encoding/json.(*Decoder).Decode+0x8d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/encoding/json/stream.go:57 # 0xc4a5f3 net/rpc/jsonrpc.(*serverCodec).ReadRequestHeader+0x73 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/rpc/jsonrpc/server.go:66 # 0x77c97b net/rpc.(*Server).readRequestHeader+0x6b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/rpc/server.go:576 # 0x77c449 net/rpc.(*Server).readRequest+0x59 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/rpc/server.go:543 # 0x77ba19 net/rpc.(*Server).ServeCodec+0x89 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/rpc/server.go:462 # 0x788dfc github.com/couchbase/cbauth/revrpc.(*Service).Run+0x79c godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:147 # 0x78970b github.com/couchbase/cbauth/revrpc.BabysitService+0x5b godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:243 # 0x47b710 github.com/couchbase/cbauth.runRPCForSvc+0x100 godeps/src/github.com/couchbase/cbauth/default.go:59 # 0x47cea4 github.com/couchbase/cbauth.startDefault.func1+0x34 godeps/src/github.com/couchbase/cbauth/default.go:66 1 @ 0x43488a 0x42f14b 0x42e8a9 0x7a53d8 0x7a5444 0x7a6c81 0x7b7d10 0xc21efc 0xc2255c 0x78862d 0x773a3a 0x773688 0x7731ce 0xc4a5f4 0x77c97c 0x77c44a 0x77ba1a 0x788dfd 0x78970c 0x9bae9e 0x5839c5 0x4659d1 # 0x42e8a8 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53d7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5443 net.(*pollDesc).waitRead+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:78 # 0x7a6c80 net.(*netFD).Read+0x1a0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:243 # 0x7b7d0f net.(*conn).Read+0x6f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/net.go:173 # 0xc21efb bufio.(*Reader).fill+0x10b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:97 # 0xc2255b bufio.(*Reader).Read+0x1bb /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/bufio/bufio.go:209 # 0x78862c github.com/couchbase/cbauth/revrpc.(*minirwc).Read+0x4c godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:101 # 0x773a39 encoding/json.(*Decoder).refill+0xf9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/encoding/json/stream.go:152 # 0x773687 encoding/json.(*Decoder).readValue+0x197 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/encoding/json/stream.go:128 # 0x7731cd encoding/json.(*Decoder).Decode+0x8d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/encoding/json/stream.go:57 # 0xc4a5f3 net/rpc/jsonrpc.(*serverCodec).ReadRequestHeader+0x73 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/rpc/jsonrpc/server.go:66 # 0x77c97b net/rpc.(*Server).readRequestHeader+0x6b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/rpc/server.go:576 # 0x77c449 net/rpc.(*Server).readRequest+0x59 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/rpc/server.go:543 # 0x77ba19 net/rpc.(*Server).ServeCodec+0x89 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/rpc/server.go:462 # 0x788dfc github.com/couchbase/cbauth/revrpc.(*Service).Run+0x79c godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:147 # 0x78970b github.com/couchbase/cbauth/revrpc.BabysitService+0x5b godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:243 # 0x9bae9d github.com/couchbase/cbauth/service.RegisterManager+0xbd godeps/src/github.com/couchbase/cbauth/service/revrpc.go:146 # 0x5839c4 github.com/couchbase/indexing/secondary/indexer.(*ServiceMgr).registerWithServer+0x1d4 goproj/src/github.com/couchbase/indexing/secondary/indexer/rebalance_service_manager.go:274 1 @ 0x43488a 0x42f14b 0x42e8a9 0x7a53d8 0x7a5444 0x7a6c81 0x7b7d10 0xcd9a19 0xcda4fa 0xcd93b0 0xbd05ae 0x4659d1 # 0x42e8a8 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53d7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5443 net.(*pollDesc).waitRead+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:78 # 0x7a6c80 net.(*netFD).Read+0x1a0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:243 # 0x7b7d0f net.(*conn).Read+0x6f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/net.go:173 # 0xcd9a18 github.com/couchbase/indexing/secondary/transport.fullRead+0x88 goproj/src/github.com/couchbase/indexing/secondary/transport/transport.go:189 # 0xcda4f9 github.com/couchbase/indexing/secondary/transport.Receive+0x89 goproj/src/github.com/couchbase/indexing/secondary/transport/util.go:60 # 0xcd93af github.com/couchbase/indexing/secondary/transport.(*TransportPacket).Receive+0x6f goproj/src/github.com/couchbase/indexing/secondary/transport/transport.go:119 # 0xbd05ad github.com/couchbase/indexing/secondary/dataport.doReceive+0x28d goproj/src/github.com/couchbase/indexing/secondary/dataport/server.go:563 1 @ 0x43488a 0x42f14b 0x42e8a9 0x7a53d8 0x7a5444 0x7a8598 0x7c317e 0x7c16a9 0x73178f 0x7309ae 0x7306e4 0x62eba7 0x4659d1 # 0x42e8a8 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53d7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5443 net.(*pollDesc).waitRead+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:78 # 0x7a8597 net.(*netFD).accept+0x237 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:419 # 0x7c317d net.(*TCPListener).accept+0x2d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/tcpsock_posix.go:132 # 0x7c16a8 net.(*TCPListener).AcceptTCP+0x48 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/tcpsock.go:209 # 0x73178e net/http.tcpKeepAliveListener.Accept+0x2e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:2608 # 0x7309ad net/http.(*Server).Serve+0x1cd /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:2273 # 0x7306e3 net/http.(*Server).ListenAndServe+0xb3 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:2219 # 0x62eba6 github.com/couchbase/indexing/secondary/indexer.(*indexer).initHttpServer.func1+0x216 goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:373 1 @ 0x43488a 0x42f14b 0x42e8a9 0x7a53d8 0x7a5444 0x7a8598 0x7c317e 0x7c18c9 0x6e9d07 0x7309ae 0x7303f9 0x62f1e0 0x4659d1 # 0x42e8a8 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53d7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5443 net.(*pollDesc).waitRead+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:78 # 0x7a8597 net.(*netFD).accept+0x237 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:419 # 0x7c317d net.(*TCPListener).accept+0x2d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/tcpsock_posix.go:132 # 0x7c18c8 net.(*TCPListener).Accept+0x48 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/tcpsock.go:222 # 0x6e9d06 crypto/tls.(*listener).Accept+0x36 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/crypto/tls/tls.go:52 # 0x7309ad net/http.(*Server).Serve+0x1cd /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:2273 # 0x7303f8 net/http.Serve+0x98 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:2091 # 0x62f1df github.com/couchbase/indexing/secondary/indexer.(*indexer).initHttpServer.func3+0x4ef goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:445 1 @ 0x43488a 0x42f14b 0x42e8a9 0x7a53d8 0x7a5444 0x7a8598 0x7c317e 0x7c18c9 0xbc781d 0x4659d1 # 0x42e8a8 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53d7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5443 net.(*pollDesc).waitRead+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:78 # 0x7a8597 net.(*netFD).accept+0x237 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:419 # 0x7c317d net.(*TCPListener).accept+0x2d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/tcpsock_posix.go:132 # 0x7c18c8 net.(*TCPListener).Accept+0x48 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/tcpsock.go:222 # 0xbc781c github.com/couchbase/indexing/secondary/queryport.(*Server).listener+0xac goproj/src/github.com/couchbase/indexing/secondary/queryport/server.go:150 1 @ 0x43488a 0x42f14b 0x42e8a9 0x7a53d8 0x7a5444 0x7a8598 0x7c317e 0x7c18c9 0xbcff5c 0x4659d1 # 0x42e8a8 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53d7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5443 net.(*pollDesc).waitRead+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:78 # 0x7a8597 net.(*netFD).accept+0x237 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:419 # 0x7c317d net.(*TCPListener).accept+0x2d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/tcpsock_posix.go:132 # 0x7c18c8 net.(*TCPListener).Accept+0x48 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/tcpsock.go:222 # 0xbcff5b github.com/couchbase/indexing/secondary/dataport.listener+0x4b goproj/src/github.com/couchbase/indexing/secondary/dataport/server.go:515 1 @ 0x43488a 0x42f14b 0x42e8a9 0x7a53d8 0x7a5444 0x7a8598 0x7c317e 0x7c18c9 0xc959c2 0x4659d1 # 0x42e8a8 net.runtime_pollWait+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/netpoll.go:160 # 0x7a53d7 net.(*pollDesc).wait+0x37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:73 # 0x7a5443 net.(*pollDesc).waitRead+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_poll_runtime.go:78 # 0x7a8597 net.(*netFD).accept+0x237 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/fd_unix.go:419 # 0x7c317d net.(*TCPListener).accept+0x2d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/tcpsock_posix.go:132 # 0x7c18c8 net.(*TCPListener).Accept+0x48 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/tcpsock.go:222 # 0xc959c1 github.com/couchbase/gometa/common.(*PeerListener).listen+0x61 goproj/src/github.com/couchbase/gometa/common/peerListener.go:135 1 @ 0x43488a 0x43497e 0x40c3af 0x40c075 0x65ff94 0x4659d1 # 0x65ff93 github.com/couchbase/indexing/secondary/logging.PeriodicProfile.func1+0x53 goproj/src/github.com/couchbase/indexing/secondary/logging/logging.go:199 1 @ 0x43488a 0x43497e 0x40c3af 0x40c0b5 0x4d212b 0x4659d1 # 0x4d212a github.com/couchbase/indexing/secondary/indexer.(*clustMgrAgent).run+0xba goproj/src/github.com/couchbase/indexing/secondary/indexer/cluster_manager_agent.go:99 1 @ 0x43488a 0x43497e 0x40c3af 0x40c0b5 0x4df97f 0x4659d1 # 0x4df97e github.com/couchbase/indexing/secondary/indexer.(*compactionManager).run+0xae goproj/src/github.com/couchbase/indexing/secondary/indexer/compaction_manager.go:731 1 @ 0x43488a 0x43497e 0x40c3af 0x40c0b5 0x4e04cb 0x4659d1 # 0x4e04ca github.com/couchbase/indexing/secondary/indexer.(*cpuCollector).runCollectStats+0x10a goproj/src/github.com/couchbase/indexing/secondary/indexer/cpu.go:70 1 @ 0x43488a 0x43497e 0x40c3af 0x40c0b5 0x4e1d16 0x4659d1 # 0x4e1d15 github.com/couchbase/indexing/secondary/indexer.(*DDLServiceMgr).run+0x85 goproj/src/github.com/couchbase/indexing/secondary/indexer/ddl_service_manager.go:159 1 @ 0x43488a 0x43497e 0x40c3af 0x40c0b5 0x541413 0x4659d1 # 0x541412 github.com/couchbase/indexing/secondary/indexer.(*kvSender).run+0x62 goproj/src/github.com/couchbase/indexing/secondary/indexer/kv_sender.go:85 1 @ 0x43488a 0x43497e 0x40c3af 0x40c0b5 0x562322 0x4659d1 # 0x562321 github.com/couchbase/indexing/secondary/indexer.(*mutationMgr).run+0xd1 goproj/src/github.com/couchbase/indexing/secondary/indexer/mutation_manager.go:122 1 @ 0x43488a 0x43497e 0x40c3af 0x40c0b5 0x5833c7 0x4659d1 # 0x5833c6 github.com/couchbase/indexing/secondary/indexer.(*ServiceMgr).run+0x66 goproj/src/github.com/couchbase/indexing/secondary/indexer/rebalance_service_manager.go:222 1 @ 0x43488a 0x43497e 0x40c3af 0x40c0b5 0x5b7d43 0x4659d1 # 0x5b7d42 github.com/couchbase/indexing/secondary/indexer.(*scanCoordinator).run+0x62 goproj/src/github.com/couchbase/indexing/secondary/indexer/scan_coordinator.go:122 1 @ 0x43488a 0x43497e 0x40c3af 0x40c0b5 0x5b7eb0 0x4659d1 # 0x5b7eaf github.com/couchbase/indexing/secondary/indexer.(*scanCoordinator).listenSnapshot+0x5f goproj/src/github.com/couchbase/indexing/secondary/indexer/scan_coordinator.go:140 1 @ 0x43488a 0x43497e 0x40c3af 0x40c0b5 0x5e2d13 0x4659d1 # 0x5e2d12 github.com/couchbase/indexing/secondary/indexer.(*settingsManager).run+0x62 goproj/src/github.com/couchbase/indexing/secondary/indexer/settings.go:243 1 @ 0x43488a 0x43497e 0x40c3af 0x40c0b5 0x5f2394 0x4659d1 # 0x5f2393 github.com/couchbase/indexing/secondary/indexer.(*statsManager).run+0x63 goproj/src/github.com/couchbase/indexing/secondary/indexer/stats_manager.go:1309 1 @ 0x43488a 0x43497e 0x40c3af 0x40c0b5 0x5f3503 0x4659d1 # 0x5f3502 github.com/couchbase/indexing/secondary/indexer.(*storageMgr).run+0x62 goproj/src/github.com/couchbase/indexing/secondary/indexer/storage_manager.go:153 1 @ 0x43488a 0x43497e 0x40c3af 0x40c0b5 0x5fd775 0x4659d1 # 0x5fd774 github.com/couchbase/indexing/secondary/indexer.(*mutationStreamReader).listenSupvCmd+0x84 goproj/src/github.com/couchbase/indexing/secondary/indexer/stream_reader.go:203 1 @ 0x43488a 0x43497e 0x40c3af 0x40c0b5 0x610653 0x4659d1 # 0x610652 github.com/couchbase/indexing/secondary/indexer.(*timekeeper).run+0x62 goproj/src/github.com/couchbase/indexing/secondary/indexer/timekeeper.go:112 1 @ 0x43488a 0x43497e 0x40c3af 0x40c0b5 0xcb2e3d 0x4659d1 # 0xcb2e3c github.com/couchbase/gometa/protocol.(*Leader).listen+0xcc goproj/src/github.com/couchbase/gometa/protocol/leader.go:472 1 @ 0x43488a 0x43497e 0x44f9e1 0x4896b1 0x4659d1 # 0x44f9e0 time.Sleep+0xe0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/time.go:59 # 0x4896b0 github.com/couchbase/indexing/secondary/common.pollForDeletedBuckets+0x40 goproj/src/github.com/couchbase/indexing/secondary/common/dcp_seqno.go:341 1 @ 0x43488a 0x43497e 0x44f9e1 0x53d113 0x4659d1 # 0x44f9e0 time.Sleep+0xe0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/time.go:59 # 0x53d112 github.com/couchbase/indexing/secondary/indexer.(*indexer).monitorMemUsage+0x282 goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:6306 1 @ 0x43488a 0x43497e 0x44f9e1 0x53fae2 0x4659d1 # 0x44f9e0 time.Sleep+0xe0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/time.go:59 # 0x53fae1 github.com/couchbase/indexing/secondary/indexer.(*indexer).logMemstats+0x191 goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:6586 1 @ 0x43488a 0x43497e 0x44f9e1 0x58b526 0x4659d1 # 0x44f9e0 time.Sleep+0xe0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/time.go:59 # 0x58b525 github.com/couchbase/indexing/secondary/indexer.(*ServiceMgr).rebalanceJanitor+0x45 goproj/src/github.com/couchbase/indexing/secondary/indexer/rebalance_service_manager.go:1204 1 @ 0x43488a 0x43497e 0x44f9e1 0x5f2850 0x4659d1 # 0x44f9e0 time.Sleep+0xe0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/time.go:59 # 0x5f284f github.com/couchbase/indexing/secondary/indexer.(*statsManager).runStatsDumpLogger+0xbf goproj/src/github.com/couchbase/indexing/secondary/indexer/stats_manager.go:1362 1 @ 0x43488a 0x43497e 0x44f9e1 0x99049f 0x4659d1 # 0x44f9e0 time.Sleep+0xe0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/time.go:59 # 0x99049e github.com/couchbase/indexing/secondary/manager.(*builder).run+0x30e goproj/src/github.com/couchbase/indexing/secondary/manager/lifecycle.go:2759 1 @ 0x43488a 0x443f25 0x442cbc 0x48444a 0x4659d1 # 0x484449 github.com/couchbase/indexing/secondary/common.(*ClusterInfoClient).watchClusterChanges+0x429 goproj/src/github.com/couchbase/indexing/secondary/common/cluster_info.go:684 1 @ 0x43488a 0x443f25 0x442cbc 0x4da19d 0x4659d1 # 0x4da19c github.com/couchbase/indexing/secondary/indexer.(*compactionDaemon).loop+0x2dc goproj/src/github.com/couchbase/indexing/secondary/indexer/compaction_manager.go:117 1 @ 0x43488a 0x443f25 0x442cbc 0x4e6611 0x4659d1 # 0x4e6610 github.com/couchbase/indexing/secondary/indexer.(*DDLServiceMgr).processCreateCommand+0x500 goproj/src/github.com/couchbase/indexing/secondary/indexer/ddl_service_manager.go:759 1 @ 0x43488a 0x443f25 0x442cbc 0x509f01 0x50733c 0x406e91 0x434424 0x4659d1 # 0x509f00 github.com/couchbase/indexing/secondary/indexer.(*indexer).run+0x290 goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:568 # 0x50733b github.com/couchbase/indexing/secondary/indexer.NewIndexer+0x1f2b goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:327 # 0x406e90 main.main+0x12e0 goproj/src/github.com/couchbase/indexing/secondary/cmd/indexer/main.go:165 # 0x434423 runtime.main+0x1f3 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/proc.go:183 1 @ 0x43488a 0x443f25 0x442cbc 0x50a1d8 0x4659d1 # 0x50a1d7 github.com/couchbase/indexing/secondary/indexer.(*indexer).listenAdminMsgs+0x2c7 goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:602 1 @ 0x43488a 0x443f25 0x442cbc 0x50a7e4 0x4659d1 # 0x50a7e3 github.com/couchbase/indexing/secondary/indexer.(*indexer).listenWorkerMsgs+0x2c3 goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:661 1 @ 0x43488a 0x443f25 0x442cbc 0x5625a2 0x4659d1 # 0x5625a1 github.com/couchbase/indexing/secondary/indexer.(*mutationMgr).handleWorkerMsgs+0x161 goproj/src/github.com/couchbase/indexing/secondary/indexer/mutation_manager.go:146 1 @ 0x43488a 0x443f25 0x442cbc 0x562a47 0x4659d1 # 0x562a46 github.com/couchbase/indexing/secondary/indexer.(*mutationMgr).listenWorkerMsgs+0x496 goproj/src/github.com/couchbase/indexing/secondary/indexer/mutation_manager.go:169 1 @ 0x43488a 0x443f25 0x442cbc 0x58f10c 0x583c24 0x9bb36e 0x9ba6f9 0x9bab24 0x9bbd10 0x462f1c 0x6812a8 0x680bc4 0x77abf8 0x4659d1 # 0x58f10b github.com/couchbase/indexing/secondary/indexer.(*ServiceMgr).wait+0x44b goproj/src/github.com/couchbase/indexing/secondary/indexer/rebalance_service_manager.go:1630 # 0x583c23 github.com/couchbase/indexing/secondary/indexer.(*ServiceMgr).GetTaskList+0x133 goproj/src/github.com/couchbase/indexing/secondary/indexer/rebalance_service_manager.go:305 # 0x9bb36d github.com/couchbase/cbauth/service.serviceAPI.GetTaskList.func1+0x7d godeps/src/github.com/couchbase/cbauth/service/revrpc.go:99 # 0x9ba6f8 github.com/couchbase/cbauth/service.withTimeout+0x88 godeps/src/github.com/couchbase/cbauth/service/revrpc.go:46 # 0x9bab23 github.com/couchbase/cbauth/service.serviceAPI.GetTaskList+0xe3 godeps/src/github.com/couchbase/cbauth/service/revrpc.go:103 # 0x9bbd0f github.com/couchbase/cbauth/service.(*serviceAPI).GetTaskList+0x8f :11 # 0x462f1b runtime.call64+0x4b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/asm_amd64.s:480 # 0x6812a7 reflect.Value.call+0x5c7 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/reflect/value.go:434 # 0x680bc3 reflect.Value.Call+0xa3 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/reflect/value.go:302 # 0x77abf7 net/rpc.(*service).call+0x147 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/rpc/server.go:383 1 @ 0x43488a 0x443f25 0x442cbc 0x58f10c 0x5843b7 0x9bb14e 0x9ba6f9 0x9ba9b4 0x9bbc00 0x462f1c 0x6812a8 0x680bc4 0x77abf8 0x4659d1 # 0x58f10b github.com/couchbase/indexing/secondary/indexer.(*ServiceMgr).wait+0x44b goproj/src/github.com/couchbase/indexing/secondary/indexer/rebalance_service_manager.go:1630 # 0x5843b6 github.com/couchbase/indexing/secondary/indexer.(*ServiceMgr).GetCurrentTopology+0x136 goproj/src/github.com/couchbase/indexing/secondary/indexer/rebalance_service_manager.go:355 # 0x9bb14d github.com/couchbase/cbauth/service.serviceAPI.GetCurrentTopology.func1+0x7d godeps/src/github.com/couchbase/cbauth/service/revrpc.go:76 # 0x9ba6f8 github.com/couchbase/cbauth/service.withTimeout+0x88 godeps/src/github.com/couchbase/cbauth/service/revrpc.go:46 # 0x9ba9b3 github.com/couchbase/cbauth/service.serviceAPI.GetCurrentTopology+0xe3 godeps/src/github.com/couchbase/cbauth/service/revrpc.go:80 # 0x9bbbff github.com/couchbase/cbauth/service.(*serviceAPI).GetCurrentTopology+0x8f :10 # 0x462f1b runtime.call64+0x4b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/asm_amd64.s:480 # 0x6812a7 reflect.Value.call+0x5c7 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/reflect/value.go:434 # 0x680bc3 reflect.Value.Call+0xa3 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/reflect/value.go:302 # 0x77abf7 net/rpc.(*service).call+0x147 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/rpc/server.go:383 1 @ 0x43488a 0x443f25 0x442cbc 0x5fd6ce 0x4659d1 # 0x5fd6cd github.com/couchbase/indexing/secondary/indexer.(*mutationStreamReader).run+0x32d goproj/src/github.com/couchbase/indexing/secondary/indexer/stream_reader.go:162 1 @ 0x43488a 0x443f25 0x442cbc 0x5fe945 0x4659d1 # 0x5fe944 github.com/couchbase/indexing/secondary/indexer.(*mutationStreamReader).syncWorker+0x1a4 goproj/src/github.com/couchbase/indexing/secondary/indexer/stream_reader.go:355 1 @ 0x43488a 0x443f25 0x442cbc 0x75ab72 0x4659d1 # 0x75ab71 github.com/couchbase/cbauth/cbauthimpl.(*certNotifier).loop+0x141 godeps/src/github.com/couchbase/cbauth/cbauthimpl/impl.go:214 1 @ 0x43488a 0x443f25 0x442cbc 0x940669 0x94004c 0x940a95 0x5930e4 0x4659d1 # 0x940668 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x5d8 godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0x94004b github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x5b godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0x940a94 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x5930e3 github.com/couchbase/indexing/secondary/indexer.(*ServiceMgr).listenMoveIndex+0xf3 goproj/src/github.com/couchbase/indexing/secondary/indexer/rebalance_service_manager.go:2116 1 @ 0x43488a 0x443f25 0x442cbc 0x940669 0x94004c 0x940a95 0x646465 0x4946e2 0x64662d 0x4659d1 # 0x940668 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x5d8 godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0x94004b github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x5b godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0x940a94 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x646464 github.com/couchbase/indexing/secondary/indexer.NewSettingsManager.func1.1+0x94 goproj/src/github.com/couchbase/indexing/secondary/indexer/settings.go:82 # 0x4946e1 github.com/couchbase/indexing/secondary/common.(*RetryHelper).Run+0x51 goproj/src/github.com/couchbase/indexing/secondary/common/retry_helper.go:36 # 0x64662c github.com/couchbase/indexing/secondary/indexer.NewSettingsManager.func1+0xbc goproj/src/github.com/couchbase/indexing/secondary/indexer/settings.go:86 1 @ 0x43488a 0x443f25 0x442cbc 0x97c5bf 0x4659d1 # 0x97c5be github.com/couchbase/indexing/secondary/manager.(*LifecycleMgr).processRequest+0x46e goproj/src/github.com/couchbase/indexing/secondary/manager/lifecycle.go:277 1 @ 0x43488a 0x443f25 0x442cbc 0x990164 0x4659d1 # 0x990163 github.com/couchbase/indexing/secondary/manager.(*janitor).run+0x4d3 goproj/src/github.com/couchbase/indexing/secondary/manager/lifecycle.go:2698 1 @ 0x43488a 0x443f25 0x442cbc 0x994088 0x4659d1 # 0x994087 github.com/couchbase/indexing/secondary/manager.(*updator).run+0x277 goproj/src/github.com/couchbase/indexing/secondary/manager/lifecycle.go:3119 1 @ 0x43488a 0x443f25 0x442cbc 0x998525 0x4659d1 # 0x998524 github.com/couchbase/indexing/secondary/manager.(*IndexManager).monitorBucket+0x2c4 goproj/src/github.com/couchbase/indexing/secondary/manager/manager.go:666 1 @ 0x43488a 0x443f25 0x442cbc 0xbce714 0x4659d1 # 0xbce713 github.com/couchbase/indexing/secondary/dataport.(*Server).genServer+0x963 goproj/src/github.com/couchbase/indexing/secondary/dataport/server.go:338 1 @ 0x43488a 0x443f25 0x442cbc 0xcb7006 0x4659d1 # 0xcb7005 github.com/couchbase/gometa/protocol.(*LeaderServer).listenFollower+0x4b5 goproj/src/github.com/couchbase/gometa/protocol/leaderServer.go:186 1 @ 0x43488a 0x443f25 0x442cbc 0xcb8817 0xcb6839 0xcc71a6 0xcc690f 0x4659d1 # 0xcb8816 github.com/couchbase/gometa/protocol.(*LeaderServer).processRequest+0x656 goproj/src/github.com/couchbase/gometa/protocol/leaderServer.go:384 # 0xcb6838 github.com/couchbase/gometa/protocol.RunLeaderServerWithCustomHandler+0x5c8 goproj/src/github.com/couchbase/gometa/protocol/leaderServer.go:130 # 0xcc71a5 github.com/couchbase/gometa/server.(*EmbeddedServer).runOnce+0x165 goproj/src/github.com/couchbase/gometa/server/embeddedServer.go:483 # 0xcc690e github.com/couchbase/gometa/server.(*EmbeddedServer).run+0x2e goproj/src/github.com/couchbase/gometa/server/embeddedServer.go:385 1 @ 0x4659d1 1 @ 0x6bd495 0x6ba315 0x6b8ab9 0x46f62f 0x46ce99 0x49b555 0x4659d1 # 0x6bd494 syscall.Syscall+0x4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/syscall/asm_linux_amd64.s:18 # 0x6ba314 syscall.read+0x54 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/syscall/zsyscall_linux_amd64.go:783 # 0x6b8ab8 syscall.Read+0x48 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/syscall/syscall_unix.go:161 # 0x46f62e os.(*File).read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/os/file_unix.go:228 # 0x46ce98 os.(*File).Read+0x58 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/os/file.go:101 # 0x49b554 github.com/couchbase/indexing/secondary/common.ExitOnStdinClose+0x54 goproj/src/github.com/couchbase/indexing/secondary/common/util.go:355 1 @ 0xa188bf 0xa186c0 0xa15371 0xd4a116 0xd4a495 0x72e654 0x72fa3f 0x73052d 0x72ced7 0x4659d1 # 0xa188be runtime/pprof.writeRuntimeProfile+0x9e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/pprof/pprof.go:614 # 0xa186bf runtime/pprof.writeGoroutine+0x9f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/pprof/pprof.go:576 # 0xa15370 runtime/pprof.(*Profile).WriteTo+0x340 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/pprof/pprof.go:298 # 0xd4a115 net/http/pprof.handler.ServeHTTP+0x1a5 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/pprof/pprof.go:209 # 0xd4a494 net/http/pprof.Index+0x204 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/pprof/pprof.go:221 # 0x72e653 net/http.HandlerFunc.ServeHTTP+0x43 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:1726 # 0x72fa3e net/http.(*ServeMux).ServeHTTP+0x7e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:2022 # 0x73052c net/http.serverHandler.ServeHTTP+0x7c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:2202 # 0x72ced6 net/http.(*conn).serve+0x4b6 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/net/http/server.go:1579