goroutine profile: total 1386 1 @ 0x7759d6 0x77577d 0x771865 0x62b2ef 0x62b45e 0x5fd4c1 0x5fec2d 0x5ff5fa 0x5fd017 0x42ae61 # 0x7759d6 runtime/pprof.writeRuntimeProfile+0xd6 /usr/local/go/src/runtime/pprof/pprof.go:540 # 0x77577d runtime/pprof.writeGoroutine+0x9d /usr/local/go/src/runtime/pprof/pprof.go:502 # 0x771865 runtime/pprof.(*Profile).WriteTo+0xd5 /usr/local/go/src/runtime/pprof/pprof.go:229 # 0x62b2ef net/http/pprof.handler.ServeHTTP+0x35f /usr/local/go/src/net/http/pprof/pprof.go:169 # 0x62b45e net/http/pprof.Index+0x15e /usr/local/go/src/net/http/pprof/pprof.go:181 # 0x5fd4c1 net/http.HandlerFunc.ServeHTTP+0x41 /usr/local/go/src/net/http/server.go:1265 # 0x5fec2d net/http.(*ServeMux).ServeHTTP+0x17d /usr/local/go/src/net/http/server.go:1541 # 0x5ff5fa net/http.serverHandler.ServeHTTP+0x19a /usr/local/go/src/net/http/server.go:1703 # 0x5fd017 net/http.(*conn).serve+0xb57 /usr/local/go/src/net/http/server.go:1204 1 @ 0x417835 0x4178a8 0x405a62 0x4055cb 0x402348 0x417573 0x42ae61 # 0x402348 main.main+0x14c8 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/main/main.go:142 # 0x417573 runtime.main+0xf3 /usr/local/go/src/runtime/proc.go:63 1 @ 0x417835 0x4178a8 0x4176ce 0x42ae61 # 0x417835 runtime.gopark+0x105 /usr/local/go/src/runtime/proc.go:131 # 0x4178a8 runtime.goparkunlock+0x48 /usr/local/go/src/runtime/proc.go:136 # 0x4176ce runtime.forcegchelper+0xce /usr/local/go/src/runtime/proc.go:99 1 @ 0x417835 0x4178a8 0x410e7c 0x42ae61 # 0x417835 runtime.gopark+0x105 /usr/local/go/src/runtime/proc.go:131 # 0x4178a8 runtime.goparkunlock+0x48 /usr/local/go/src/runtime/proc.go:136 # 0x410e7c runtime.bgsweep+0xbc /usr/local/go/src/runtime/mgc0.go:98 1 @ 0x417835 0x4178a8 0x41047a 0x42ae61 # 0x417835 runtime.gopark+0x105 /usr/local/go/src/runtime/proc.go:131 # 0x4178a8 runtime.goparkunlock+0x48 /usr/local/go/src/runtime/proc.go:136 # 0x41047a runtime.runfinq+0xba /usr/local/go/src/runtime/malloc.go:727 1 @ 0x40ea02 0x41edda 0x42ae61 # 0x40ea02 runtime.notetsleepg+0x52 /usr/local/go/src/runtime/lock_futex.go:202 # 0x41edda runtime.timerproc+0xfa /usr/local/go/src/runtime/time.go:207 1 @ 0x417835 0x4140f0 0x413588 0x57d3f7 0x57d463 0x57f11f 0x59182c 0x63d0ae 0x63d60c 0x756ebc 0x55e9c1 0x55e07b 0x7c6dfc 0x753c87 0x753912 0x752e3d 0x7575a8 0x757cf0 0x61b033 0x61c5f6 0x42ae61 # 0x57d3f7 net.(*pollDesc).Wait+0x47 /usr/local/go/src/net/fd_poll_runtime.go:84 # 0x57d463 net.(*pollDesc).WaitRead+0x43 /usr/local/go/src/net/fd_poll_runtime.go:89 # 0x57f11f net.(*netFD).Read+0x40f /usr/local/go/src/net/fd_unix.go:242 # 0x59182c net.(*conn).Read+0xdc /usr/local/go/src/net/net.go:121 # 0x63d0ae bufio.(*Reader).fill+0x1ce /usr/local/go/src/bufio/bufio.go:97 # 0x63d60c bufio.(*Reader).Read+0x26c /usr/local/go/src/bufio/bufio.go:174 # 0x756ebc github.com/couchbase/cbauth/revrpc.(*minirwc).Read+0x5c /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:100 # 0x55e9c1 encoding/json.(*Decoder).readValue+0x5e1 /usr/local/go/src/encoding/json/stream.go:124 # 0x55e07b encoding/json.(*Decoder).Decode+0x7b /usr/local/go/src/encoding/json/stream.go:44 # 0x7c6dfc net/rpc/jsonrpc.(*serverCodec).ReadRequestHeader+0x9c /usr/local/go/src/net/rpc/jsonrpc/server.go:66 # 0x753c87 net/rpc.(*Server).readRequestHeader+0x97 /usr/local/go/src/net/rpc/server.go:575 # 0x753912 net/rpc.(*Server).readRequest+0xa2 /usr/local/go/src/net/rpc/server.go:542 # 0x752e3d net/rpc.(*Server).ServeCodec+0x6d /usr/local/go/src/net/rpc/server.go:461 # 0x7575a8 github.com/couchbase/cbauth/revrpc.(*Service).Run+0x6c8 /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:146 # 0x757cf0 github.com/couchbase/cbauth/revrpc.BabysitService+0x80 /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:242 # 0x61b033 github.com/couchbase/cbauth.runRPCForSvc+0x193 /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/cbauth/default.go:56 # 0x61c5f6 github.com/couchbase/cbauth.func·004+0x36 /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/cbauth/default.go:63 261 @ 0x417835 0x419adc 0x418f72 0x620018 0x42ae61 # 0x620018 github.com/couchbase/goxdcr/gen_server.(*GenServer).run+0x998 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/gen_server/gen_server.go:78 2 @ 0x417835 0x419adc 0x418f72 0x60ccae 0x42ae61 # 0x60ccae net/http.(*persistConn).readLoop+0x9ce /usr/local/go/src/net/http/transport.go:928 9 @ 0x417835 0x419adc 0x418f72 0x60d48d 0x42ae61 # 0x60d48d net/http.(*persistConn).writeLoop+0x41d /usr/local/go/src/net/http/transport.go:945 112 @ 0x417835 0x419adc 0x418f72 0x6bf40e 0x6c5efb 0x42ae61 # 0x6bf40e github.com/couchbase/goxdcr/parts.(*requestBuffer).flowControl+0x13e /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:202 # 0x6c5efb github.com/couchbase/goxdcr/parts.(*XmemNozzle).processData_batch+0x6db /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:971 1 @ 0x6e62f5 0x6e396e 0x6e29b8 0x4bf8ee 0x4bcdb1 0x63d0ae 0x63d806 0x49bbba 0x42ae61 # 0x6e62f5 syscall.Syscall+0x5 /usr/local/go/src/syscall/asm_linux_amd64.s:22 # 0x6e396e syscall.read+0x6e /usr/local/go/src/syscall/zsyscall_linux_amd64.go:867 # 0x6e29b8 syscall.Read+0x58 /usr/local/go/src/syscall/syscall_unix.go:136 # 0x4bf8ee os.(*File).read+0x5e /usr/local/go/src/os/file_unix.go:191 # 0x4bcdb1 os.(*File).Read+0x91 /usr/local/go/src/os/file.go:95 # 0x63d0ae bufio.(*Reader).fill+0x1ce /usr/local/go/src/bufio/bufio.go:97 # 0x63d806 bufio.(*Reader).ReadByte+0x86 /usr/local/go/src/bufio/bufio.go:196 # 0x49bbba github.com/couchbase/goxdcr/replication_manager.pollStdin+0xaa /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/replication_manager/replication_manager.go:631 4 @ 0x417835 0x419adc 0x418f72 0x68340f 0x42ae61 # 0x68340f github.com/couchbase/goxdcr/supervisor.(*GenericSupervisor).supervising+0x46f /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/supervisor/generic_supervisor.go:185 2 @ 0x417835 0x419adc 0x418f72 0x5d8ed5 0x5d8520 0x5d9534 0x48c4ad 0x42ae61 # 0x5d8ed5 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x995 /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:259 # 0x5d8520 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x80 /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:216 # 0x5d9534 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x64 /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:342 # 0x48c4ad github.com/couchbase/goxdcr/replication_manager.(*MetakvChangeListener).observeChildren+0xad /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/replication_manager/metakv_change_listener.go:66 1 @ 0x417835 0x419adc 0x418f72 0x496f34 0x42ae61 # 0x496f34 github.com/couchbase/goxdcr/replication_manager.(*replicationManager).checkReplicationStatus+0x214 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/replication_manager/replication_manager.go:205 1 @ 0x417835 0x419adc 0x418f72 0x4851ab 0x42ae61 # 0x4851ab github.com/couchbase/goxdcr/replication_manager.(*Adminport).Start+0x75b /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/replication_manager/adminport.go:102 2 @ 0x417835 0x4140f0 0x413588 0x57d3f7 0x57d463 0x57f11f 0x59182c 0x60eb8e 0x618454 0x63d0ae 0x63de57 0x749aea 0x749406 0x74969b 0x60620a 0x606135 0x60e4a5 0x55e9c1 0x55e07b 0x5dbec4 0x42ae61 # 0x57d3f7 net.(*pollDesc).Wait+0x47 /usr/local/go/src/net/fd_poll_runtime.go:84 # 0x57d463 net.(*pollDesc).WaitRead+0x43 /usr/local/go/src/net/fd_poll_runtime.go:89 # 0x57f11f net.(*netFD).Read+0x40f /usr/local/go/src/net/fd_unix.go:242 # 0x59182c net.(*conn).Read+0xdc /usr/local/go/src/net/net.go:121 # 0x60eb8e net/http.noteEOFReader.Read+0x6e /usr/local/go/src/net/http/transport.go:1270 # 0x618454 net/http.(*noteEOFReader).Read+0xd4 :125 # 0x63d0ae bufio.(*Reader).fill+0x1ce /usr/local/go/src/bufio/bufio.go:97 # 0x63de57 bufio.(*Reader).ReadSlice+0x257 /usr/local/go/src/bufio/bufio.go:295 # 0x749aea net/http/internal.readLine+0x5a /usr/local/go/src/net/http/internal/chunked.go:110 # 0x749406 net/http/internal.(*chunkedReader).beginChunk+0x46 /usr/local/go/src/net/http/internal/chunked.go:47 # 0x74969b net/http/internal.(*chunkedReader).Read+0xbb /usr/local/go/src/net/http/internal/chunked.go:77 # 0x60620a net/http.(*body).readLocked+0x7a /usr/local/go/src/net/http/transfer.go:584 # 0x606135 net/http.(*body).Read+0x115 /usr/local/go/src/net/http/transfer.go:579 # 0x60e4a5 net/http.(*bodyEOFSignal).Read+0x285 /usr/local/go/src/net/http/transport.go:1193 # 0x55e9c1 encoding/json.(*Decoder).readValue+0x5e1 /usr/local/go/src/encoding/json/stream.go:124 # 0x55e07b encoding/json.(*Decoder).Decode+0x7b /usr/local/go/src/encoding/json/stream.go:44 # 0x5dbec4 github.com/couchbase/cbauth/metakv.func·003+0x134 /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:242 1 @ 0x417835 0x4140f0 0x413588 0x57d3f7 0x57d463 0x581b1b 0x599d6e 0x599e7c 0x5ff862 0x61ec37 0x42ae61 # 0x57d3f7 net.(*pollDesc).Wait+0x47 /usr/local/go/src/net/fd_poll_runtime.go:84 # 0x57d463 net.(*pollDesc).WaitRead+0x43 /usr/local/go/src/net/fd_poll_runtime.go:89 # 0x581b1b net.(*netFD).accept+0x40b /usr/local/go/src/net/fd_unix.go:419 # 0x599d6e net.(*TCPListener).AcceptTCP+0x4e /usr/local/go/src/net/tcpsock_posix.go:234 # 0x599e7c net.(*TCPListener).Accept+0x4c /usr/local/go/src/net/tcpsock_posix.go:244 # 0x5ff862 net/http.(*Server).Serve+0x92 /usr/local/go/src/net/http/server.go:1728 # 0x61ec37 github.com/couchbase/goxdcr/adminport.func·001+0x157 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/adminport/admin_httpd.go:95 17 @ 0x417835 0x4140f0 0x413588 0x57d3f7 0x57d463 0x57f11f 0x59182c 0x4c20c1 0x4c21fd 0x5bd94e 0x56c01a 0x42ae61 # 0x57d3f7 net.(*pollDesc).Wait+0x47 /usr/local/go/src/net/fd_poll_runtime.go:84 # 0x57d463 net.(*pollDesc).WaitRead+0x43 /usr/local/go/src/net/fd_poll_runtime.go:89 # 0x57f11f net.(*netFD).Read+0x40f /usr/local/go/src/net/fd_unix.go:242 # 0x59182c net.(*conn).Read+0xdc /usr/local/go/src/net/net.go:121 # 0x4c20c1 io.ReadAtLeast+0xf1 /usr/local/go/src/io/io.go:298 # 0x4c21fd io.ReadFull+0x6d /usr/local/go/src/io/io.go:316 # 0x5bd94e github.com/couchbase/gomemcached.(*MCRequest).Receive+0xee /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/gomemcached/mc_req.go:130 # 0x56c01a github.com/couchbase/gomemcached/client.(*UprFeed).runFeed+0x16a /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/gomemcached/client/upr_feed.go:402 128 @ 0x417835 0x419adc 0x418f72 0x6bbacd 0x42ae61 # 0x6bbacd github.com/couchbase/goxdcr/parts.(*DcpNozzle).collectDcpDataChanLen+0x1ad /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:843 111 @ 0x417835 0x419adc 0x418f72 0x56c8be 0x42ae61 # 0x56c8be github.com/couchbase/gomemcached/client.(*UprFeed).runFeed+0xa0e /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/gomemcached/client/upr_feed.go:547 127 @ 0x417835 0x4178a8 0x41a7ff 0x41a58d 0x5086b3 0x509703 0x6bf8f9 0x6d5209 0x6d4bfa 0x6d16e9 0x6d7ede 0x6bf950 0x6d10b1 0x42ae61 # 0x5086b3 sync.(*Mutex).Lock+0xd3 /usr/local/go/src/sync/mutex.go:66 # 0x509703 sync.(*RWMutex).Lock+0x33 /usr/local/go/src/sync/rwmutex.go:82 # 0x6bf8f9 github.com/couchbase/goxdcr/parts.(*requestBuffer).modSlot+0xf9 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:251 # 0x6d5209 github.com/couchbase/goxdcr/parts.(*XmemNozzle).onSetMetaConnRepaired+0xb9 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2034 # 0x6d4bfa github.com/couchbase/goxdcr/parts.(*XmemNozzle).repairConn+0x4da /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2016 # 0x6d16e9 github.com/couchbase/goxdcr/parts.(*XmemNozzle).checkTimeout+0x3e9 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1767 # 0x6d7ede github.com/couchbase/goxdcr/parts.*XmemNozzle.(github.com/couchbase/goxdcr/parts.checkTimeout)·fm+0x4e /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1741 # 0x6bf950 github.com/couchbase/goxdcr/parts.(*requestBuffer).modSlot+0x150 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:257 # 0x6d10b1 github.com/couchbase/goxdcr/parts.(*XmemNozzle).check+0x8a1 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1743 16 @ 0x417835 0x419adc 0x418f72 0x6c62f8 0x42ae61 # 0x6c62f8 github.com/couchbase/goxdcr/parts.(*XmemNozzle).processData_batch+0xad8 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:958 109 @ 0x417835 0x4178a8 0x41a7ff 0x41a58d 0x5086b3 0x509703 0x6bf8f9 0x6d5209 0x6d4bfa 0x6cc8c7 0x42ae61 # 0x5086b3 sync.(*Mutex).Lock+0xd3 /usr/local/go/src/sync/mutex.go:66 # 0x509703 sync.(*RWMutex).Lock+0x33 /usr/local/go/src/sync/rwmutex.go:82 # 0x6bf8f9 github.com/couchbase/goxdcr/parts.(*requestBuffer).modSlot+0xf9 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:251 # 0x6d5209 github.com/couchbase/goxdcr/parts.(*XmemNozzle).onSetMetaConnRepaired+0xb9 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2034 # 0x6d4bfa github.com/couchbase/goxdcr/parts.(*XmemNozzle).repairConn+0x4da /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2016 # 0x6cc8c7 github.com/couchbase/goxdcr/parts.(*XmemNozzle).receiveResponse+0x657 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1501 2 @ 0x417835 0x419adc 0x418f72 0x67d1c1 0x42ae61 # 0x67d1c1 github.com/couchbase/goxdcr/pipeline_svc.(*TopologyChangeDetectorSvc).watch+0x211 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/topology_change_detector.go:89 48 @ 0x417835 0x419adc 0x418f72 0x78eb33 0x42ae61 # 0x78eb33 github.com/couchbase/goxdcr/component.(*AsyncComponentEventListenerImpl).start+0x3e3 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/component/async_listener.go:68 128 @ 0x417835 0x419adc 0x418f72 0x6d07f3 0x42ae61 # 0x6d07f3 github.com/couchbase/goxdcr/parts.(*XmemNozzle).selfMonitor+0x1ca3 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1673 2 @ 0x417835 0x419adc 0x418f72 0x66b7f9 0x42ae61 # 0x66b7f9 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).updateStats+0x549 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:272 128 @ 0x417835 0x419adc 0x418f72 0x569483 0x42ae61 # 0x569483 github.com/couchbase/gomemcached/client.sendCommands+0x283 /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/gomemcached/client/upr_feed.go:148 111 @ 0x417835 0x4178a8 0x405054 0x404883 0x6d5fe7 0x6c5313 0x79aa71 0x6b6dc0 0x42ae61 # 0x6d5fe7 github.com/couchbase/goxdcr/parts.(*XmemNozzle).writeToDataChan+0x87 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2090 # 0x6c5313 github.com/couchbase/goxdcr/parts.(*XmemNozzle).Receive+0x823 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:926 # 0x79aa71 github.com/couchbase/goxdcr/connector.(*Router).Forward+0x221 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/connector/router.go:62 # 0x6b6dc0 github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0x1c80 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:431 7 @ 0x417835 0x4140f0 0x413588 0x57d3f7 0x57d463 0x57f11f 0x59182c 0x60eb8e 0x618454 0x63d0ae 0x63d2f0 0x60c384 0x42ae61 # 0x57d3f7 net.(*pollDesc).Wait+0x47 /usr/local/go/src/net/fd_poll_runtime.go:84 # 0x57d463 net.(*pollDesc).WaitRead+0x43 /usr/local/go/src/net/fd_poll_runtime.go:89 # 0x57f11f net.(*netFD).Read+0x40f /usr/local/go/src/net/fd_unix.go:242 # 0x59182c net.(*conn).Read+0xdc /usr/local/go/src/net/net.go:121 # 0x60eb8e net/http.noteEOFReader.Read+0x6e /usr/local/go/src/net/http/transport.go:1270 # 0x618454 net/http.(*noteEOFReader).Read+0xd4 :125 # 0x63d0ae bufio.(*Reader).fill+0x1ce /usr/local/go/src/bufio/bufio.go:97 # 0x63d2f0 bufio.(*Reader).Peek+0xf0 /usr/local/go/src/bufio/bufio.go:132 # 0x60c384 net/http.(*persistConn).readLoop+0xa4 /usr/local/go/src/net/http/transport.go:842 17 @ 0x417835 0x419adc 0x418f72 0x6b7474 0x42ae61 # 0x6b7474 github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0x2334 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:367 2 @ 0x417835 0x419adc 0x418f72 0x668cb6 0x42ae61 # 0x668cb6 github.com/couchbase/goxdcr/pipeline_svc.(*PipelineSupervisor).monitorPipelineHealth+0x376 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/pipeline_supervisor.go:137 2 @ 0x417835 0x419adc 0x418f72 0x663e39 0x42ae61 # 0x663e39 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).checkpointing+0x419 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:633 2 @ 0x417835 0x419adc 0x418f72 0x666ea3 0x42ae61 # 0x666ea3 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).massCheckVBOpaquesJob+0x253 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:882 2 @ 0x417835 0x4140f0 0x413588 0x57d3f7 0x57d463 0x57f11f 0x59182c 0x5f6cfb 0x4c284e 0x63d0ae 0x63de57 0x63ded2 0x74538e 0x7450df 0x5f20eb 0x5f93ff 0x5fcb5e 0x42ae61 # 0x57d3f7 net.(*pollDesc).Wait+0x47 /usr/local/go/src/net/fd_poll_runtime.go:84 # 0x57d463 net.(*pollDesc).WaitRead+0x43 /usr/local/go/src/net/fd_poll_runtime.go:89 # 0x57f11f net.(*netFD).Read+0x40f /usr/local/go/src/net/fd_unix.go:242 # 0x59182c net.(*conn).Read+0xdc /usr/local/go/src/net/net.go:121 # 0x5f6cfb net/http.(*liveSwitchReader).Read+0xab /usr/local/go/src/net/http/server.go:214 # 0x4c284e io.(*LimitedReader).Read+0xce /usr/local/go/src/io/io.go:408 # 0x63d0ae bufio.(*Reader).fill+0x1ce /usr/local/go/src/bufio/bufio.go:97 # 0x63de57 bufio.(*Reader).ReadSlice+0x257 /usr/local/go/src/bufio/bufio.go:295 # 0x63ded2 bufio.(*Reader).ReadLine+0x62 /usr/local/go/src/bufio/bufio.go:324 # 0x74538e net/textproto.(*Reader).readLineSlice+0x9e /usr/local/go/src/net/textproto/reader.go:55 # 0x7450df net/textproto.(*Reader).ReadLine+0x4f /usr/local/go/src/net/textproto/reader.go:36 # 0x5f20eb net/http.ReadRequest+0xcb /usr/local/go/src/net/http/request.go:598 # 0x5f93ff net/http.(*conn).readRequest+0x26f /usr/local/go/src/net/http/server.go:586 # 0x5fcb5e net/http.(*conn).serve+0x69e /usr/local/go/src/net/http/server.go:1162 4 @ 0x417835 0x4178a8 0x41e7a5 0x6644ce 0x42ae61 # 0x6644ce github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).performCkpt+0x66e /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:699 16 @ 0x417835 0x4178a8 0x41a7ff 0x41a58d 0x5086b3 0x509703 0x6bf8f9 0x6ccd08 0x42ae61 # 0x5086b3 sync.(*Mutex).Lock+0xd3 /usr/local/go/src/sync/mutex.go:66 # 0x509703 sync.(*RWMutex).Lock+0x33 /usr/local/go/src/sync/rwmutex.go:82 # 0x6bf8f9 github.com/couchbase/goxdcr/parts.(*requestBuffer).modSlot+0xf9 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:251 # 0x6ccd08 github.com/couchbase/goxdcr/parts.(*XmemNozzle).receiveResponse+0xa98 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1512 2 @ 0x417835 0x4178a8 0x41a7ff 0x41a58d 0x5086b3 0x509703 0x6bf8f9 0x6d5209 0x6d4bfa 0x6d353a 0x6ca2ae 0x6d1b11 0x6d7e6e 0x6bf950 0x6ccd08 0x42ae61 # 0x5086b3 sync.(*Mutex).Lock+0xd3 /usr/local/go/src/sync/mutex.go:66 # 0x509703 sync.(*RWMutex).Lock+0x33 /usr/local/go/src/sync/rwmutex.go:82 # 0x6bf8f9 github.com/couchbase/goxdcr/parts.(*requestBuffer).modSlot+0xf9 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:251 # 0x6d5209 github.com/couchbase/goxdcr/parts.(*XmemNozzle).onSetMetaConnRepaired+0xb9 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2034 # 0x6d4bfa github.com/couchbase/goxdcr/parts.(*XmemNozzle).repairConn+0x4da /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2016 # 0x6d353a github.com/couchbase/goxdcr/parts.(*XmemNozzle).writeToClient+0x47a /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1909 # 0x6ca2ae github.com/couchbase/goxdcr/parts.(*XmemNozzle).sendSingleSetMeta+0x3ae /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1355 # 0x6d1b11 github.com/couchbase/goxdcr/parts.(*XmemNozzle).resend+0x261 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1794 # 0x6d7e6e github.com/couchbase/goxdcr/parts.*XmemNozzle.(github.com/couchbase/goxdcr/parts.resend)·fm+0x4e /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1512 # 0x6bf950 github.com/couchbase/goxdcr/parts.(*requestBuffer).modSlot+0x150 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:257 # 0x6ccd08 github.com/couchbase/goxdcr/parts.(*XmemNozzle).receiveResponse+0xa98 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1512 1 @ 0x417835 0x419adc 0x418f72 0x6ce7d4 0x42ae61 # 0x6ce7d4 github.com/couchbase/goxdcr/parts.(*XmemNozzle).receiveResponse+0x2564 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1485 1 @ 0x417835 0x419adc 0x418f72 0x6d12ef 0x42ae61 # 0x6d12ef github.com/couchbase/goxdcr/parts.(*XmemNozzle).check+0xadf /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1725