goroutine profile: total 2623 1 @ 0x64d046 0x64cded 0x648e65 0x64778f 0x647cdd 0x54e4bd 0x54ee8a 0x54c8a7 0x42b9a1 # 0x64d046 runtime/pprof.writeRuntimeProfile+0xd6 /usr/local/go/src/runtime/pprof/pprof.go:540 # 0x64cded runtime/pprof.writeGoroutine+0x9d /usr/local/go/src/runtime/pprof/pprof.go:502 # 0x648e65 runtime/pprof.(*Profile).WriteTo+0xd5 /usr/local/go/src/runtime/pprof/pprof.go:229 # 0x64778f net/http/pprof.handler.ServeHTTP+0x35f /usr/local/go/src/net/http/pprof/pprof.go:169 # 0x647cdd net/http/pprof.(*handler).ServeHTTP+0xbd :1 # 0x54e4bd net/http.(*ServeMux).ServeHTTP+0x17d /usr/local/go/src/net/http/server.go:1541 # 0x54ee8a net/http.serverHandler.ServeHTTP+0x19a /usr/local/go/src/net/http/server.go:1703 # 0x54c8a7 net/http.(*conn).serve+0xb57 /usr/local/go/src/net/http/server.go:1204 1 @ 0x418035 0x4180a8 0x4062e2 0x405e4b 0x402795 0x417d73 0x42b9a1 # 0x402795 main.main+0xa05 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/cmd/projector/main.go:108 # 0x417d73 runtime.main+0xf3 /usr/local/go/src/runtime/proc.go:63 1 @ 0x418035 0x4180a8 0x417ece 0x42b9a1 # 0x418035 runtime.gopark+0x105 /usr/local/go/src/runtime/proc.go:131 # 0x4180a8 runtime.goparkunlock+0x48 /usr/local/go/src/runtime/proc.go:136 # 0x417ece runtime.forcegchelper+0xce /usr/local/go/src/runtime/proc.go:99 1 @ 0x418035 0x4180a8 0x41170c 0x42b9a1 # 0x418035 runtime.gopark+0x105 /usr/local/go/src/runtime/proc.go:131 # 0x4180a8 runtime.goparkunlock+0x48 /usr/local/go/src/runtime/proc.go:136 # 0x41170c runtime.bgsweep+0xbc /usr/local/go/src/runtime/mgc0.go:98 1 @ 0x418035 0x4180a8 0x410d0a 0x42b9a1 # 0x418035 runtime.gopark+0x105 /usr/local/go/src/runtime/proc.go:131 # 0x4180a8 runtime.goparkunlock+0x48 /usr/local/go/src/runtime/proc.go:136 # 0x410d0a runtime.runfinq+0xba /usr/local/go/src/runtime/malloc.go:727 1 @ 0x40f292 0x41c135 0x647fff 0x42b9a1 # 0x647fff os/signal.loop+0x1f /usr/local/go/src/os/signal/signal_unix.go:21 1 @ 0x418035 0x4148f0 0x413d88 0x58c7b7 0x58c823 0x58e4df 0x5a0c7c 0x67efde 0x67f53c 0x579bdc 0x5e60f1 0x5e57ab 0x6f295c 0x571157 0x570de2 0x57030d 0x57a2c8 0x57aa10 0x471703 0x472ff6 0x42b9a1 # 0x58c7b7 net.(*pollDesc).Wait+0x47 /usr/local/go/src/net/fd_poll_runtime.go:84 # 0x58c823 net.(*pollDesc).WaitRead+0x43 /usr/local/go/src/net/fd_poll_runtime.go:89 # 0x58e4df net.(*netFD).Read+0x40f /usr/local/go/src/net/fd_unix.go:242 # 0x5a0c7c net.(*conn).Read+0xdc /usr/local/go/src/net/net.go:121 # 0x67efde bufio.(*Reader).fill+0x1ce /usr/local/go/src/bufio/bufio.go:97 # 0x67f53c bufio.(*Reader).Read+0x26c /usr/local/go/src/bufio/bufio.go:174 # 0x579bdc github.com/couchbase/cbauth/revrpc.(*minirwc).Read+0x5c /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:100 # 0x5e60f1 encoding/json.(*Decoder).readValue+0x5e1 /usr/local/go/src/encoding/json/stream.go:124 # 0x5e57ab encoding/json.(*Decoder).Decode+0x7b /usr/local/go/src/encoding/json/stream.go:44 # 0x6f295c net/rpc/jsonrpc.(*serverCodec).ReadRequestHeader+0x9c /usr/local/go/src/net/rpc/jsonrpc/server.go:66 # 0x571157 net/rpc.(*Server).readRequestHeader+0x97 /usr/local/go/src/net/rpc/server.go:575 # 0x570de2 net/rpc.(*Server).readRequest+0xa2 /usr/local/go/src/net/rpc/server.go:542 # 0x57030d net/rpc.(*Server).ServeCodec+0x6d /usr/local/go/src/net/rpc/server.go:461 # 0x57a2c8 github.com/couchbase/cbauth/revrpc.(*Service).Run+0x6c8 /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:146 # 0x57aa10 github.com/couchbase/cbauth/revrpc.BabysitService+0x80 /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:242 # 0x471703 github.com/couchbase/cbauth.runRPCForSvc+0x193 /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/cbauth/default.go:56 # 0x472ff6 github.com/couchbase/cbauth.func·004+0x36 /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/cbauth/default.go:63 1 @ 0x40f292 0x41f8fa 0x42b9a1 # 0x40f292 runtime.notetsleepg+0x52 /usr/local/go/src/runtime/lock_futex.go:202 # 0x41f8fa runtime.timerproc+0xfa /usr/local/go/src/runtime/time.go:207 1 @ 0x418035 0x4180a8 0x41f2c5 0x47a0c1 0x42b9a1 # 0x47a0c1 github.com/couchbase/indexing/secondary/common.pollForDeletedBuckets+0x41 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/common/dcp_seqno.go:281 1 @ 0x532a95 0x5301be 0x52f0f8 0x464ece 0x462451 0x485b09 0x42b9a1 # 0x532a95 syscall.Syscall+0x5 /usr/local/go/src/syscall/asm_linux_amd64.s:22 # 0x5301be syscall.read+0x6e /usr/local/go/src/syscall/zsyscall_linux_amd64.go:867 # 0x52f0f8 syscall.Read+0x58 /usr/local/go/src/syscall/syscall_unix.go:136 # 0x464ece os.(*File).read+0x5e /usr/local/go/src/os/file_unix.go:191 # 0x462451 os.(*File).Read+0x91 /usr/local/go/src/os/file.go:95 # 0x485b09 github.com/couchbase/indexing/secondary/common.ExitOnStdinClose+0x99 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/common/util.go:316 1 @ 0x418035 0x41a2dc 0x419772 0x6079ab 0x6070d0 0x607c94 0x489a48 0x42b9a1 # 0x6079ab github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x8bb /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:284 # 0x6070d0 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x80 /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:244 # 0x607c94 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x64 /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:367 # 0x489a48 github.com/couchbase/indexing/secondary/common.func·012+0x68 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/common/settings.go:49 1 @ 0x418035 0x41a2dc 0x419772 0x55c53e 0x42b9a1 # 0x55c53e net/http.(*persistConn).readLoop+0x9ce /usr/local/go/src/net/http/transport.go:928 2 @ 0x418035 0x41a2dc 0x419772 0x55cd1d 0x42b9a1 # 0x55cd1d net/http.(*persistConn).writeLoop+0x41d /usr/local/go/src/net/http/transport.go:945 1 @ 0x418035 0x4180a8 0x4062e2 0x405e7b 0x49e9d4 0x42b9a1 # 0x49e9d4 github.com/couchbase/indexing/secondary/projector.(*Projector).mainAdminPort+0xe64 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/adminport.go:64 1 @ 0x418035 0x41a2dc 0x419772 0x484d1e 0x4a3f45 0x4c4c18 0x4d5fff 0x42b9a1 # 0x484d1e github.com/couchbase/indexing/secondary/common.FailsafeOp+0x24e /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/common/util.go:155 # 0x4a3f45 github.com/couchbase/indexing/secondary/projector.(*Feed).Ping+0x175 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/feed.go:346 # 0x4c4c18 github.com/couchbase/indexing/secondary/projector.(*Projector).GetFeed+0xe8 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/projector.go:173 # 0x4d5fff github.com/couchbase/indexing/secondary/projector.(*Projector).watcherDameon+0x11f /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/watcher.go:18 1 @ 0x418035 0x4148f0 0x413d88 0x58c7b7 0x58c823 0x590edb 0x5a91be 0x5a92cc 0x54f0f2 0x654f51 0x42b9a1 # 0x58c7b7 net.(*pollDesc).Wait+0x47 /usr/local/go/src/net/fd_poll_runtime.go:84 # 0x58c823 net.(*pollDesc).WaitRead+0x43 /usr/local/go/src/net/fd_poll_runtime.go:89 # 0x590edb net.(*netFD).accept+0x40b /usr/local/go/src/net/fd_unix.go:419 # 0x5a91be net.(*TCPListener).AcceptTCP+0x4e /usr/local/go/src/net/tcpsock_posix.go:234 # 0x5a92cc net.(*TCPListener).Accept+0x4c /usr/local/go/src/net/tcpsock_posix.go:244 # 0x54f0f2 net/http.(*Server).Serve+0x92 /usr/local/go/src/net/http/server.go:1728 # 0x654f51 github.com/couchbase/indexing/secondary/adminport.func·002+0x151 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/adminport/admin_httpd.go:197 1 @ 0x418035 0x4180a8 0x4066bc 0x405e4b 0x49d6cd 0x42b9a1 # 0x49d6cd github.com/couchbase/indexing/secondary/logging.func·001+0x7d /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/logging/logging.go:196 1 @ 0x418035 0x4148f0 0x413d88 0x58c7b7 0x58c823 0x58e4df 0x5a0c7c 0x55e41e 0x567ce4 0x67efde 0x67fd87 0x6b099a 0x6b02b6 0x6b054b 0x555a9a 0x5559c5 0x55dd35 0x5e60f1 0x5e57ab 0x60a624 0x42b9a1 # 0x58c7b7 net.(*pollDesc).Wait+0x47 /usr/local/go/src/net/fd_poll_runtime.go:84 # 0x58c823 net.(*pollDesc).WaitRead+0x43 /usr/local/go/src/net/fd_poll_runtime.go:89 # 0x58e4df net.(*netFD).Read+0x40f /usr/local/go/src/net/fd_unix.go:242 # 0x5a0c7c net.(*conn).Read+0xdc /usr/local/go/src/net/net.go:121 # 0x55e41e net/http.noteEOFReader.Read+0x6e /usr/local/go/src/net/http/transport.go:1270 # 0x567ce4 net/http.(*noteEOFReader).Read+0xd4 :125 # 0x67efde bufio.(*Reader).fill+0x1ce /usr/local/go/src/bufio/bufio.go:97 # 0x67fd87 bufio.(*Reader).ReadSlice+0x257 /usr/local/go/src/bufio/bufio.go:295 # 0x6b099a net/http/internal.readLine+0x5a /usr/local/go/src/net/http/internal/chunked.go:110 # 0x6b02b6 net/http/internal.(*chunkedReader).beginChunk+0x46 /usr/local/go/src/net/http/internal/chunked.go:47 # 0x6b054b net/http/internal.(*chunkedReader).Read+0xbb /usr/local/go/src/net/http/internal/chunked.go:77 # 0x555a9a net/http.(*body).readLocked+0x7a /usr/local/go/src/net/http/transfer.go:584 # 0x5559c5 net/http.(*body).Read+0x115 /usr/local/go/src/net/http/transfer.go:579 # 0x55dd35 net/http.(*bodyEOFSignal).Read+0x285 /usr/local/go/src/net/http/transport.go:1193 # 0x5e60f1 encoding/json.(*Decoder).readValue+0x5e1 /usr/local/go/src/encoding/json/stream.go:124 # 0x5e57ab encoding/json.(*Decoder).Decode+0x7b /usr/local/go/src/encoding/json/stream.go:44 # 0x60a624 github.com/couchbase/cbauth/metakv.func·003+0x134 /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:267 2282 @ 0x418035 0x41a2dc 0x419772 0x484f47 0x496624 0x4d4b8c 0x4d15aa 0x42b9a1 # 0x484f47 github.com/couchbase/indexing/secondary/common.FailsafeOpAsync+0x147 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/common/util.go:173 # 0x496624 github.com/couchbase/indexing/secondary/dataport.(*RouterEndpoint).Send+0x114 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/dataport/endpoint.go:125 # 0x4d4b8c github.com/couchbase/indexing/secondary/projector.(*VbucketRoutine).broadcast2Endpoints+0xec /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/vbucket.go:361 # 0x4d15aa github.com/couchbase/indexing/secondary/projector.(*VbucketRoutine).run+0xbaa /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/vbucket.go:184 1 @ 0x418035 0x41a2dc 0x419772 0x484d1e 0x49628d 0x4b0ff8 0x4ab8e3 0x4a8135 0x4a5ba7 0x42b9a1 # 0x484d1e github.com/couchbase/indexing/secondary/common.FailsafeOp+0x24e /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/common/util.go:155 # 0x49628d github.com/couchbase/indexing/secondary/dataport.(*RouterEndpoint).Ping+0x15d /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/dataport/endpoint.go:105 # 0x4b0ff8 github.com/couchbase/indexing/secondary/projector.(*Feed).repairEndpoints+0x4e8 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/feed.go:1112 # 0x4ab8e3 github.com/couchbase/indexing/secondary/projector.(*Feed).restartVbuckets+0xd3 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/feed.go:773 # 0x4a8135 github.com/couchbase/indexing/secondary/projector.(*Feed).handleCommand+0x3e5 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/feed.go:570 # 0x4a5ba7 github.com/couchbase/indexing/secondary/projector.(*Feed).genServer+0x227 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/feed.go:463 1 @ 0x418035 0x4148f0 0x413d88 0x58c7b7 0x58c883 0x58fdae 0x5a0d8c 0x61f161 0x61e32b 0x498910 0x49a713 0x4974cc 0x42b9a1 # 0x58c7b7 net.(*pollDesc).Wait+0x47 /usr/local/go/src/net/fd_poll_runtime.go:84 # 0x58c883 net.(*pollDesc).WaitWrite+0x43 /usr/local/go/src/net/fd_poll_runtime.go:93 # 0x58fdae net.(*netFD).Write+0x5ee /usr/local/go/src/net/fd_unix.go:335 # 0x5a0d8c net.(*conn).Write+0xdc /usr/local/go/src/net/net.go:129 # 0x61f161 github.com/couchbase/indexing/secondary/transport.Send+0x3b1 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/transport/util.go:23 # 0x61e32b github.com/couchbase/indexing/secondary/transport.(*TransportPacket).Send+0x16b /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/transport/transport.go:106 # 0x498910 github.com/couchbase/indexing/secondary/dataport.(*endpointBuffers).flushBuffers+0x2f0 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/dataport/endpoint_buffer.go:43 # 0x49a713 github.com/couchbase/indexing/secondary/dataport.func·009+0x243 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/dataport/endpoint.go:180 # 0x4974cc github.com/couchbase/indexing/secondary/dataport.(*RouterEndpoint).run+0x91c /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/dataport/endpoint.go:213 1 @ 0x418035 0x41a2dc 0x419772 0x484f47 0x496b84 0x4bb7bc 0x42b9a1 # 0x484f47 github.com/couchbase/indexing/secondary/common.FailsafeOpAsync+0x147 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/common/util.go:173 # 0x496b84 github.com/couchbase/indexing/secondary/dataport.(*RouterEndpoint).WaitForExit+0x84 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/dataport/endpoint.go:148 # 0x4bb7bc github.com/couchbase/indexing/secondary/projector.(*Feed).watchEndpoint+0x3c /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/feed.go:1743 9 @ 0x418035 0x4180a8 0x4066bc 0x405e4b 0x5ca880 0x42b9a1 # 0x5ca880 github.com/couchbase/indexing/secondary/dcp.(*DcpFeed).genServer+0x140 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/dcp/upr.go:236 5 @ 0x418035 0x41a2dc 0x419772 0x484f47 0x4cfd72 0x4d761f 0x42b9a1 # 0x484f47 github.com/couchbase/indexing/secondary/common.FailsafeOpAsync+0x147 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/common/util.go:173 # 0x4cfd72 github.com/couchbase/indexing/secondary/projector.(*VbucketRoutine).SyncPulse+0xe2 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/vbucket.go:87 # 0x4d761f github.com/couchbase/indexing/secondary/projector.func·009+0x7f /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/kvdata.go:216 1 @ 0x418035 0x41a2dc 0x419772 0x484f47 0x4cfc67 0x4c1ff2 0x4be58a 0x42b9a1 # 0x484f47 github.com/couchbase/indexing/secondary/common.FailsafeOpAsync+0x147 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/common/util.go:173 # 0x4cfc67 github.com/couchbase/indexing/secondary/projector.(*VbucketRoutine).Event+0x127 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/vbucket.go:80 # 0x4c1ff2 github.com/couchbase/indexing/secondary/projector.(*KVData).scatterMutation+0x1282 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/kvdata.go:392 # 0x4be58a github.com/couchbase/indexing/secondary/projector.(*KVData).runScatter+0x3ea /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/kvdata.go:199 264 @ 0x418035 0x41a2dc 0x419772 0x484f47 0x496624 0x4d4b8c 0x4d4138 0x4d1223 0x42b9a1 # 0x484f47 github.com/couchbase/indexing/secondary/common.FailsafeOpAsync+0x147 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/common/util.go:173 # 0x496624 github.com/couchbase/indexing/secondary/dataport.(*RouterEndpoint).Send+0x114 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/dataport/endpoint.go:125 # 0x4d4b8c github.com/couchbase/indexing/secondary/projector.(*VbucketRoutine).broadcast2Endpoints+0xec /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/vbucket.go:361 # 0x4d4138 github.com/couchbase/indexing/secondary/projector.(*VbucketRoutine).handleEvent+0x7d8 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/vbucket.go:316 # 0x4d1223 github.com/couchbase/indexing/secondary/projector.(*VbucketRoutine).run+0x823 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/vbucket.go:253 5 @ 0x418035 0x4148f0 0x413d88 0x58c7b7 0x58c823 0x58e4df 0x5a0c7c 0x4da051 0x4da18d 0x66e2ce 0x5f4b30 0x42b9a1 # 0x58c7b7 net.(*pollDesc).Wait+0x47 /usr/local/go/src/net/fd_poll_runtime.go:84 # 0x58c823 net.(*pollDesc).WaitRead+0x43 /usr/local/go/src/net/fd_poll_runtime.go:89 # 0x58e4df net.(*netFD).Read+0x40f /usr/local/go/src/net/fd_unix.go:242 # 0x5a0c7c net.(*conn).Read+0xdc /usr/local/go/src/net/net.go:121 # 0x4da051 io.ReadAtLeast+0xf1 /usr/local/go/src/io/io.go:298 # 0x4da18d io.ReadFull+0x6d /usr/local/go/src/io/io.go:316 # 0x66e2ce github.com/couchbase/indexing/secondary/dcp/transport.(*MCRequest).Receive+0xee /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/dcp/transport/mc_req.go:130 # 0x5f4b30 github.com/couchbase/indexing/secondary/dcp/transport/client.(*DcpFeed).doReceive+0x1b0 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/dcp/transport/client/dcp_feed.go:882 4 @ 0x418035 0x4180a8 0x4058d4 0x405103 0x5f51f7 0x42b9a1 # 0x5f51f7 github.com/couchbase/indexing/secondary/dcp/transport/client.(*DcpFeed).doReceive+0x877 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/dcp/transport/client/dcp_feed.go:899 3 @ 0x418035 0x4148f0 0x413d88 0x58c7b7 0x58c823 0x58e4df 0x5a0c7c 0x54658b 0x4da7de 0x67efde 0x67fd87 0x67fe02 0x67aa2e 0x67a77f 0x54197b 0x548c8f 0x54c3ee 0x42b9a1 # 0x58c7b7 net.(*pollDesc).Wait+0x47 /usr/local/go/src/net/fd_poll_runtime.go:84 # 0x58c823 net.(*pollDesc).WaitRead+0x43 /usr/local/go/src/net/fd_poll_runtime.go:89 # 0x58e4df net.(*netFD).Read+0x40f /usr/local/go/src/net/fd_unix.go:242 # 0x5a0c7c net.(*conn).Read+0xdc /usr/local/go/src/net/net.go:121 # 0x54658b net/http.(*liveSwitchReader).Read+0xab /usr/local/go/src/net/http/server.go:214 # 0x4da7de io.(*LimitedReader).Read+0xce /usr/local/go/src/io/io.go:408 # 0x67efde bufio.(*Reader).fill+0x1ce /usr/local/go/src/bufio/bufio.go:97 # 0x67fd87 bufio.(*Reader).ReadSlice+0x257 /usr/local/go/src/bufio/bufio.go:295 # 0x67fe02 bufio.(*Reader).ReadLine+0x62 /usr/local/go/src/bufio/bufio.go:324 # 0x67aa2e net/textproto.(*Reader).readLineSlice+0x9e /usr/local/go/src/net/textproto/reader.go:55 # 0x67a77f net/textproto.(*Reader).ReadLine+0x4f /usr/local/go/src/net/textproto/reader.go:36 # 0x54197b net/http.ReadRequest+0xcb /usr/local/go/src/net/http/request.go:598 # 0x548c8f net/http.(*conn).readRequest+0x26f /usr/local/go/src/net/http/server.go:586 # 0x54c3ee net/http.(*conn).serve+0x69e /usr/local/go/src/net/http/server.go:1162 2 @ 0x418035 0x41a2dc 0x419772 0x484f47 0x496624 0x4d4b8c 0x4d811e 0x4d1271 0x42b9a1 # 0x484f47 github.com/couchbase/indexing/secondary/common.FailsafeOpAsync+0x147 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/common/util.go:173 # 0x496624 github.com/couchbase/indexing/secondary/dataport.(*RouterEndpoint).Send+0x114 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/dataport/endpoint.go:125 # 0x4d4b8c github.com/couchbase/indexing/secondary/projector.(*VbucketRoutine).broadcast2Endpoints+0xec /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/vbucket.go:361 # 0x4d811e github.com/couchbase/indexing/secondary/projector.func·012+0x30e /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/vbucket.go:162 # 0x4d1271 github.com/couchbase/indexing/secondary/projector.(*VbucketRoutine).run+0x871 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/vbucket.go:260 5 @ 0x418035 0x41a2dc 0x419772 0x484f47 0x4cfc67 0x4c20f5 0x4be58a 0x42b9a1 # 0x484f47 github.com/couchbase/indexing/secondary/common.FailsafeOpAsync+0x147 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/common/util.go:173 # 0x4cfc67 github.com/couchbase/indexing/secondary/projector.(*VbucketRoutine).Event+0x127 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/vbucket.go:80 # 0x4c20f5 github.com/couchbase/indexing/secondary/projector.(*KVData).scatterMutation+0x1385 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/kvdata.go:400 # 0x4be58a github.com/couchbase/indexing/secondary/projector.(*KVData).runScatter+0x3ea /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/kvdata.go:199 4 @ 0x418035 0x4180a8 0x4058d4 0x405103 0x5ecbc7 0x5ec430 0x42b9a1 # 0x5ecbc7 github.com/couchbase/indexing/secondary/dcp/transport/client.(*DcpFeed).handlePacket+0x677 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/dcp/transport/client/dcp_feed.go:349 # 0x5ec430 github.com/couchbase/indexing/secondary/dcp/transport/client.(*DcpFeed).genServer+0x1830 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/dcp/transport/client/dcp_feed.go:237 1 @ 0x418035 0x41a2dc 0x419772 0x484d1e 0x4a1847 0x4c9057 0x49f116 0x42b9a1 # 0x484d1e github.com/couchbase/indexing/secondary/common.FailsafeOp+0x24e /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/common/util.go:155 # 0x4a1847 github.com/couchbase/indexing/secondary/projector.(*Feed).RestartVbuckets+0x217 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/feed.go:174 # 0x4c9057 github.com/couchbase/indexing/secondary/projector.(*Projector).doRestartVbuckets+0x767 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/projector.go:388 # 0x49f116 github.com/couchbase/indexing/secondary/projector.(*Projector).handleRequest+0x206 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/adminport.go:96 5 @ 0x418035 0x41a2dc 0x419772 0x5ec4cf 0x42b9a1 # 0x5ec4cf github.com/couchbase/indexing/secondary/dcp/transport/client.(*DcpFeed).genServer+0x18cf /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/dcp/transport/client/dcp_feed.go:165 3 @ 0x418035 0x41a2dc 0x419772 0x484f47 0x4cfc67 0x4c2394 0x4be58a 0x42b9a1 # 0x484f47 github.com/couchbase/indexing/secondary/common.FailsafeOpAsync+0x147 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/common/util.go:173 # 0x4cfc67 github.com/couchbase/indexing/secondary/projector.(*VbucketRoutine).Event+0x127 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/vbucket.go:80 # 0x4c2394 github.com/couchbase/indexing/secondary/projector.(*KVData).scatterMutation+0x1624 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/kvdata.go:408 # 0x4be58a github.com/couchbase/indexing/secondary/projector.(*KVData).runScatter+0x3ea /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/kvdata.go:199 1 @ 0x418035 0x4180a8 0x4066bc 0x405e4b 0x653d5e 0x654d55 0x54cd51 0x54e4bd 0x54ee8a 0x54c8a7 0x42b9a1 # 0x653d5e github.com/couchbase/indexing/secondary/adminport.(*httpServer).systemHandler+0xb5e /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/adminport/admin_httpd.go:290 # 0x654d55 github.com/couchbase/indexing/secondary/adminport.*httpServer.(github.com/couchbase/indexing/secondary/adminport.systemHandler)·fm+0x45 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/adminport/admin_httpd.go:89 # 0x54cd51 net/http.HandlerFunc.ServeHTTP+0x41 /usr/local/go/src/net/http/server.go:1265 # 0x54e4bd net/http.(*ServeMux).ServeHTTP+0x17d /usr/local/go/src/net/http/server.go:1541 # 0x54ee8a net/http.serverHandler.ServeHTTP+0x19a /usr/local/go/src/net/http/server.go:1703 # 0x54c8a7 net/http.(*conn).serve+0xb57 /usr/local/go/src/net/http/server.go:1204 1 @ 0x418035 0x4148f0 0x413d88 0x58c7b7 0x58c823 0x58e4df 0x5a0c7c 0x55e41e 0x567ce4 0x67efde 0x67f220 0x55bc14 0x42b9a1 # 0x58c7b7 net.(*pollDesc).Wait+0x47 /usr/local/go/src/net/fd_poll_runtime.go:84 # 0x58c823 net.(*pollDesc).WaitRead+0x43 /usr/local/go/src/net/fd_poll_runtime.go:89 # 0x58e4df net.(*netFD).Read+0x40f /usr/local/go/src/net/fd_unix.go:242 # 0x5a0c7c net.(*conn).Read+0xdc /usr/local/go/src/net/net.go:121 # 0x55e41e net/http.noteEOFReader.Read+0x6e /usr/local/go/src/net/http/transport.go:1270 # 0x567ce4 net/http.(*noteEOFReader).Read+0xd4 :125 # 0x67efde bufio.(*Reader).fill+0x1ce /usr/local/go/src/bufio/bufio.go:97 # 0x67f220 bufio.(*Reader).Peek+0xf0 /usr/local/go/src/bufio/bufio.go:132 # 0x55bc14 net/http.(*persistConn).readLoop+0xa4 /usr/local/go/src/net/http/transport.go:842 1 @ 0x42b9a1 5 @ 0x418035 0x41a2dc 0x419772 0x484f47 0x496624 0x4d4706 0x4d1223 0x42b9a1 # 0x484f47 github.com/couchbase/indexing/secondary/common.FailsafeOpAsync+0x147 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/common/util.go:173 # 0x496624 github.com/couchbase/indexing/secondary/dataport.(*RouterEndpoint).Send+0x114 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/dataport/endpoint.go:125 # 0x4d4706 github.com/couchbase/indexing/secondary/projector.(*VbucketRoutine).handleEvent+0xda6 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/vbucket.go:342 # 0x4d1223 github.com/couchbase/indexing/secondary/projector.(*VbucketRoutine).run+0x823 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/vbucket.go:253