goroutine profile: total 336 1 @ 0x778446 0x7781ed 0x7742d5 0x62c36f 0x62c4de 0x5fe2c1 0x5ffa2d 0x6003fa 0x5fde17 0x42bad1 # 0x778446 runtime/pprof.writeRuntimeProfile+0xd6 /usr/local/go/src/runtime/pprof/pprof.go:540 # 0x7781ed runtime/pprof.writeGoroutine+0x9d /usr/local/go/src/runtime/pprof/pprof.go:502 # 0x7742d5 runtime/pprof.(*Profile).WriteTo+0xd5 /usr/local/go/src/runtime/pprof/pprof.go:229 # 0x62c36f net/http/pprof.handler.ServeHTTP+0x35f /usr/local/go/src/net/http/pprof/pprof.go:169 # 0x62c4de net/http/pprof.Index+0x15e /usr/local/go/src/net/http/pprof/pprof.go:181 # 0x5fe2c1 net/http.HandlerFunc.ServeHTTP+0x41 /usr/local/go/src/net/http/server.go:1265 # 0x5ffa2d net/http.(*ServeMux).ServeHTTP+0x17d /usr/local/go/src/net/http/server.go:1541 # 0x6003fa net/http.serverHandler.ServeHTTP+0x19a /usr/local/go/src/net/http/server.go:1703 # 0x5fde17 net/http.(*conn).serve+0xb57 /usr/local/go/src/net/http/server.go:1204 1 @ 0x418185 0x4181f8 0x4063b2 0x405f1b 0x402c98 0x417ec3 0x42bad1 # 0x402c98 main.main+0x14c8 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/main/main.go:142 # 0x417ec3 runtime.main+0xf3 /usr/local/go/src/runtime/proc.go:63 1 @ 0x418185 0x4181f8 0x41801e 0x42bad1 # 0x418185 runtime.gopark+0x105 /usr/local/go/src/runtime/proc.go:131 # 0x4181f8 runtime.goparkunlock+0x48 /usr/local/go/src/runtime/proc.go:136 # 0x41801e runtime.forcegchelper+0xce /usr/local/go/src/runtime/proc.go:99 1 @ 0x418185 0x4181f8 0x4117cc 0x42bad1 # 0x418185 runtime.gopark+0x105 /usr/local/go/src/runtime/proc.go:131 # 0x4181f8 runtime.goparkunlock+0x48 /usr/local/go/src/runtime/proc.go:136 # 0x4117cc runtime.bgsweep+0xbc /usr/local/go/src/runtime/mgc0.go:98 1 @ 0x418185 0x4181f8 0x410dca 0x42bad1 # 0x418185 runtime.gopark+0x105 /usr/local/go/src/runtime/proc.go:131 # 0x4181f8 runtime.goparkunlock+0x48 /usr/local/go/src/runtime/proc.go:136 # 0x410dca runtime.runfinq+0xba /usr/local/go/src/runtime/malloc.go:727 1 @ 0x40f352 0x41c285 0x7fe68f 0x42bad1 # 0x7fe68f os/signal.loop+0x1f /usr/local/go/src/os/signal/signal_unix.go:21 1 @ 0x40f352 0x41fa4a 0x42bad1 # 0x40f352 runtime.notetsleepg+0x52 /usr/local/go/src/runtime/lock_futex.go:202 # 0x41fa4a runtime.timerproc+0xfa /usr/local/go/src/runtime/time.go:207 1 @ 0x418185 0x414a40 0x413ed8 0x57de07 0x57de73 0x57fb2f 0x59223c 0x63e12e 0x63e68c 0x75992c 0x55f3d1 0x55ea8b 0x7c99ac 0x7566f7 0x756382 0x7558ad 0x75a018 0x75a760 0x61be33 0x61d3f6 0x42bad1 # 0x57de07 net.(*pollDesc).Wait+0x47 /usr/local/go/src/net/fd_poll_runtime.go:84 # 0x57de73 net.(*pollDesc).WaitRead+0x43 /usr/local/go/src/net/fd_poll_runtime.go:89 # 0x57fb2f net.(*netFD).Read+0x40f /usr/local/go/src/net/fd_unix.go:242 # 0x59223c net.(*conn).Read+0xdc /usr/local/go/src/net/net.go:121 # 0x63e12e bufio.(*Reader).fill+0x1ce /usr/local/go/src/bufio/bufio.go:97 # 0x63e68c bufio.(*Reader).Read+0x26c /usr/local/go/src/bufio/bufio.go:174 # 0x75992c github.com/couchbase/cbauth/revrpc.(*minirwc).Read+0x5c /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:100 # 0x55f3d1 encoding/json.(*Decoder).readValue+0x5e1 /usr/local/go/src/encoding/json/stream.go:124 # 0x55ea8b encoding/json.(*Decoder).Decode+0x7b /usr/local/go/src/encoding/json/stream.go:44 # 0x7c99ac net/rpc/jsonrpc.(*serverCodec).ReadRequestHeader+0x9c /usr/local/go/src/net/rpc/jsonrpc/server.go:66 # 0x7566f7 net/rpc.(*Server).readRequestHeader+0x97 /usr/local/go/src/net/rpc/server.go:575 # 0x756382 net/rpc.(*Server).readRequest+0xa2 /usr/local/go/src/net/rpc/server.go:542 # 0x7558ad net/rpc.(*Server).ServeCodec+0x6d /usr/local/go/src/net/rpc/server.go:461 # 0x75a018 github.com/couchbase/cbauth/revrpc.(*Service).Run+0x6c8 /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:146 # 0x75a760 github.com/couchbase/cbauth/revrpc.BabysitService+0x80 /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:242 # 0x61be33 github.com/couchbase/cbauth.runRPCForSvc+0x193 /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/cbauth/default.go:56 # 0x61d3f6 github.com/couchbase/cbauth.func·004+0x36 /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/cbauth/default.go:63 6 @ 0x418185 0x41a42c 0x4198c2 0x684fef 0x42bad1 # 0x684fef github.com/couchbase/goxdcr/supervisor.(*GenericSupervisor).supervising+0x46f /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/supervisor/generic_supervisor.go:185 14 @ 0x418185 0x41a42c 0x4198c2 0x6c8175 0x42bad1 # 0x6c8175 github.com/couchbase/goxdcr/parts.(*XmemNozzle).processData_batch+0x825 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:958 3 @ 0x418185 0x41a42c 0x4198c2 0x60daae 0x42bad1 # 0x60daae net/http.(*persistConn).readLoop+0x9ce /usr/local/go/src/net/http/transport.go:928 11 @ 0x418185 0x41a42c 0x4198c2 0x60e28d 0x42bad1 # 0x60e28d net/http.(*persistConn).writeLoop+0x41d /usr/local/go/src/net/http/transport.go:945 3 @ 0x418185 0x4181f8 0x41f415 0x6654ee 0x42bad1 # 0x6654ee github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).performCkpt+0x66e /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:699 1 @ 0x6e8de5 0x6e645e 0x6e54a8 0x4c0a7e 0x4bdf41 0x63e12e 0x63e886 0x49c90a 0x42bad1 # 0x6e8de5 syscall.Syscall+0x5 /usr/local/go/src/syscall/asm_linux_amd64.s:22 # 0x6e645e syscall.read+0x6e /usr/local/go/src/syscall/zsyscall_linux_amd64.go:867 # 0x6e54a8 syscall.Read+0x58 /usr/local/go/src/syscall/syscall_unix.go:136 # 0x4c0a7e os.(*File).read+0x5e /usr/local/go/src/os/file_unix.go:191 # 0x4bdf41 os.(*File).Read+0x91 /usr/local/go/src/os/file.go:95 # 0x63e12e bufio.(*Reader).fill+0x1ce /usr/local/go/src/bufio/bufio.go:97 # 0x63e886 bufio.(*Reader).ReadByte+0x86 /usr/local/go/src/bufio/bufio.go:196 # 0x49c90a github.com/couchbase/goxdcr/replication_manager.pollStdin+0xaa /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/replication_manager/replication_manager.go:631 28 @ 0x418185 0x41a42c 0x4198c2 0x620e18 0x42bad1 # 0x620e18 github.com/couchbase/goxdcr/gen_server.(*GenServer).run+0x998 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/gen_server/gen_server.go:78 2 @ 0x418185 0x41a42c 0x4198c2 0x5d9eeb 0x5d9610 0x5da444 0x48d29d 0x42bad1 # 0x5d9eeb github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x8bb /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:284 # 0x5d9610 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x80 /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:244 # 0x5da444 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x64 /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:367 # 0x48d29d github.com/couchbase/goxdcr/replication_manager.(*MetakvChangeListener).observeChildren+0xad /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/replication_manager/metakv_change_listener.go:66 1 @ 0x418185 0x41a42c 0x4198c2 0x497c64 0x42bad1 # 0x497c64 github.com/couchbase/goxdcr/replication_manager.(*replicationManager).checkReplicationStatus+0x214 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/replication_manager/replication_manager.go:205 1 @ 0x418185 0x41a42c 0x4198c2 0x485fcb 0x42bad1 # 0x485fcb github.com/couchbase/goxdcr/replication_manager.(*Adminport).Start+0x75b /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/replication_manager/adminport.go:102 1 @ 0x418185 0x414a40 0x413ed8 0x57de07 0x57de73 0x58252b 0x59a77e 0x59a88c 0x600662 0x61fa37 0x42bad1 # 0x57de07 net.(*pollDesc).Wait+0x47 /usr/local/go/src/net/fd_poll_runtime.go:84 # 0x57de73 net.(*pollDesc).WaitRead+0x43 /usr/local/go/src/net/fd_poll_runtime.go:89 # 0x58252b net.(*netFD).accept+0x40b /usr/local/go/src/net/fd_unix.go:419 # 0x59a77e net.(*TCPListener).AcceptTCP+0x4e /usr/local/go/src/net/tcpsock_posix.go:234 # 0x59a88c net.(*TCPListener).Accept+0x4c /usr/local/go/src/net/tcpsock_posix.go:244 # 0x600662 net/http.(*Server).Serve+0x92 /usr/local/go/src/net/http/server.go:1728 # 0x61fa37 github.com/couchbase/goxdcr/adminport.func·001+0x157 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/adminport/admin_httpd.go:95 2 @ 0x418185 0x414a40 0x413ed8 0x57de07 0x57de73 0x57fb2f 0x59223c 0x60f98e 0x619254 0x63e12e 0x63eed7 0x74c55a 0x74be76 0x74c10b 0x60700a 0x606f35 0x60f2a5 0x55f3d1 0x55ea8b 0x5dcdd4 0x42bad1 # 0x57de07 net.(*pollDesc).Wait+0x47 /usr/local/go/src/net/fd_poll_runtime.go:84 # 0x57de73 net.(*pollDesc).WaitRead+0x43 /usr/local/go/src/net/fd_poll_runtime.go:89 # 0x57fb2f net.(*netFD).Read+0x40f /usr/local/go/src/net/fd_unix.go:242 # 0x59223c net.(*conn).Read+0xdc /usr/local/go/src/net/net.go:121 # 0x60f98e net/http.noteEOFReader.Read+0x6e /usr/local/go/src/net/http/transport.go:1270 # 0x619254 net/http.(*noteEOFReader).Read+0xd4 :125 # 0x63e12e bufio.(*Reader).fill+0x1ce /usr/local/go/src/bufio/bufio.go:97 # 0x63eed7 bufio.(*Reader).ReadSlice+0x257 /usr/local/go/src/bufio/bufio.go:295 # 0x74c55a net/http/internal.readLine+0x5a /usr/local/go/src/net/http/internal/chunked.go:110 # 0x74be76 net/http/internal.(*chunkedReader).beginChunk+0x46 /usr/local/go/src/net/http/internal/chunked.go:47 # 0x74c10b net/http/internal.(*chunkedReader).Read+0xbb /usr/local/go/src/net/http/internal/chunked.go:77 # 0x60700a net/http.(*body).readLocked+0x7a /usr/local/go/src/net/http/transfer.go:584 # 0x606f35 net/http.(*body).Read+0x115 /usr/local/go/src/net/http/transfer.go:579 # 0x60f2a5 net/http.(*bodyEOFSignal).Read+0x285 /usr/local/go/src/net/http/transport.go:1193 # 0x55f3d1 encoding/json.(*Decoder).readValue+0x5e1 /usr/local/go/src/encoding/json/stream.go:124 # 0x55ea8b encoding/json.(*Decoder).Decode+0x7b /usr/local/go/src/encoding/json/stream.go:44 # 0x5dcdd4 github.com/couchbase/cbauth/metakv.func·003+0x134 /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:267 26 @ 0x418185 0x41a42c 0x4198c2 0x569e93 0x42bad1 # 0x569e93 github.com/couchbase/gomemcached/client.sendCommands+0x283 /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/gomemcached/client/upr_feed.go:148 54 @ 0x418185 0x41a42c 0x4198c2 0x7915a3 0x42bad1 # 0x7915a3 github.com/couchbase/goxdcr/component.(*AsyncComponentEventListenerImpl).start+0x3e3 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/component/async_listener.go:68 25 @ 0x418185 0x414a40 0x413ed8 0x57de07 0x57de73 0x57fb2f 0x59223c 0x4c3251 0x4c338d 0x5be35e 0x56ca2a 0x42bad1 # 0x57de07 net.(*pollDesc).Wait+0x47 /usr/local/go/src/net/fd_poll_runtime.go:84 # 0x57de73 net.(*pollDesc).WaitRead+0x43 /usr/local/go/src/net/fd_poll_runtime.go:89 # 0x57fb2f net.(*netFD).Read+0x40f /usr/local/go/src/net/fd_unix.go:242 # 0x59223c net.(*conn).Read+0xdc /usr/local/go/src/net/net.go:121 # 0x4c3251 io.ReadAtLeast+0xf1 /usr/local/go/src/io/io.go:298 # 0x4c338d io.ReadFull+0x6d /usr/local/go/src/io/io.go:316 # 0x5be35e github.com/couchbase/gomemcached.(*MCRequest).Receive+0xee /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/gomemcached/mc_req.go:130 # 0x56ca2a github.com/couchbase/gomemcached/client.(*UprFeed).runFeed+0x16a /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/gomemcached/client/upr_feed.go:402 2 @ 0x418185 0x4181f8 0x41b14f 0x41aedd 0x50a729 0x660d39 0x64dc91 0x65245d 0x42bad1 # 0x50a729 sync.(*WaitGroup).Wait+0x169 /usr/local/go/src/sync/waitgroup.go:132 # 0x660d39 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).SetVBTimestamps+0xdc9 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:359 # 0x64dc91 github.com/couchbase/goxdcr/factory.(*XDCRFactory).SetStartSeqno+0x401 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/factory/xdcr_factory.go:575 # 0x65245d github.com/couchbase/goxdcr/factory.*XDCRFactory.SetStartSeqno·fm+0x4d /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/factory/xdcr_factory.go:140 15 @ 0x418185 0x41a42c 0x4198c2 0x6d0fed 0x42bad1 # 0x6d0fed github.com/couchbase/goxdcr/parts.(*XmemNozzle).receiveResponse+0x25dd /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1532 58 @ 0x418185 0x4181f8 0x41b14f 0x41aedd 0x5090c3 0x661d25 0x66291b 0x42bad1 # 0x5090c3 sync.(*Mutex).Lock+0xd3 /usr/local/go/src/sync/mutex.go:66 # 0x661d25 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).setTimestampForVB+0x905 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:384 # 0x66291b github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).startSeqnoGetter+0x90b /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:457 15 @ 0x418185 0x41a42c 0x4198c2 0x6d37da 0x42bad1 # 0x6d37da github.com/couchbase/goxdcr/parts.(*XmemNozzle).check+0x8ba /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1778 8 @ 0x418185 0x414a40 0x413ed8 0x57de07 0x57de73 0x57fb2f 0x59223c 0x60f98e 0x619254 0x63e12e 0x63e370 0x60d184 0x42bad1 # 0x57de07 net.(*pollDesc).Wait+0x47 /usr/local/go/src/net/fd_poll_runtime.go:84 # 0x57de73 net.(*pollDesc).WaitRead+0x43 /usr/local/go/src/net/fd_poll_runtime.go:89 # 0x57fb2f net.(*netFD).Read+0x40f /usr/local/go/src/net/fd_unix.go:242 # 0x59223c net.(*conn).Read+0xdc /usr/local/go/src/net/net.go:121 # 0x60f98e net/http.noteEOFReader.Read+0x6e /usr/local/go/src/net/http/transport.go:1270 # 0x619254 net/http.(*noteEOFReader).Read+0xd4 :125 # 0x63e12e bufio.(*Reader).fill+0x1ce /usr/local/go/src/bufio/bufio.go:97 # 0x63e370 bufio.(*Reader).Peek+0xf0 /usr/local/go/src/bufio/bufio.go:132 # 0x60d184 net/http.(*persistConn).readLoop+0xa4 /usr/local/go/src/net/http/transport.go:842 1 @ 0x418185 0x4181f8 0x41b14f 0x41aedd 0x509f9f 0x6c78e0 0x6d4b25 0x66d207 0x66ca17 0x42bad1 # 0x509f9f sync.(*RWMutex).RLock+0x5f /usr/local/go/src/sync/rwmutex.go:36 # 0x6c78e0 github.com/couchbase/goxdcr/parts.(*XmemNozzle).batchCount+0x40 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:949 # 0x6d4b25 github.com/couchbase/goxdcr/parts.(*XmemNozzle).StatusSummary+0x165 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1919 # 0x66d207 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).logStats+0x267 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:332 # 0x66ca17 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).updateStats+0x427 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:293 3 @ 0x418185 0x41a42c 0x4198c2 0x668153 0x42bad1 # 0x668153 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).massCheckVBOpaquesJob+0x253 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:889 15 @ 0x418185 0x41a42c 0x4198c2 0x6d2f09 0x42bad1 # 0x6d2f09 github.com/couchbase/goxdcr/parts.(*XmemNozzle).selfMonitor+0x1ba9 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1726 9 @ 0x418185 0x414a40 0x413ed8 0x57de07 0x57de73 0x57fb2f 0x59223c 0x5f7afb 0x4c39de 0x63e12e 0x63eed7 0x63ef52 0x747dfe 0x747b4f 0x5f2eeb 0x5fa1ff 0x5fd95e 0x42bad1 # 0x57de07 net.(*pollDesc).Wait+0x47 /usr/local/go/src/net/fd_poll_runtime.go:84 # 0x57de73 net.(*pollDesc).WaitRead+0x43 /usr/local/go/src/net/fd_poll_runtime.go:89 # 0x57fb2f net.(*netFD).Read+0x40f /usr/local/go/src/net/fd_unix.go:242 # 0x59223c net.(*conn).Read+0xdc /usr/local/go/src/net/net.go:121 # 0x5f7afb net/http.(*liveSwitchReader).Read+0xab /usr/local/go/src/net/http/server.go:214 # 0x4c39de io.(*LimitedReader).Read+0xce /usr/local/go/src/io/io.go:408 # 0x63e12e bufio.(*Reader).fill+0x1ce /usr/local/go/src/bufio/bufio.go:97 # 0x63eed7 bufio.(*Reader).ReadSlice+0x257 /usr/local/go/src/bufio/bufio.go:295 # 0x63ef52 bufio.(*Reader).ReadLine+0x62 /usr/local/go/src/bufio/bufio.go:324 # 0x747dfe net/textproto.(*Reader).readLineSlice+0x9e /usr/local/go/src/net/textproto/reader.go:55 # 0x747b4f net/textproto.(*Reader).ReadLine+0x4f /usr/local/go/src/net/textproto/reader.go:36 # 0x5f2eeb net/http.ReadRequest+0xcb /usr/local/go/src/net/http/request.go:598 # 0x5fa1ff net/http.(*conn).readRequest+0x26f /usr/local/go/src/net/http/server.go:586 # 0x5fd95e net/http.(*conn).serve+0x69e /usr/local/go/src/net/http/server.go:1162 2 @ 0x418185 0x4181f8 0x4059a4 0x4051d3 0x6bb827 0x65c190 0x661dd8 0x66291b 0x42bad1 # 0x6bb827 github.com/couchbase/goxdcr/parts.(*DcpNozzle).UpdateSettings+0x2b7 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:612 # 0x65c190 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).UpdateSettings+0x2b0 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:633 # 0x661dd8 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).setTimestampForVB+0x9b8 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:389 # 0x66291b github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).startSeqnoGetter+0x90b /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:457 5 @ 0x418185 0x41a42c 0x4198c2 0x6b96a4 0x42bad1 # 0x6b96a4 github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0x2334 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:367 1 @ 0x418185 0x4181f8 0x41b14f 0x41aedd 0x5090c3 0x50a113 0x6c63f8 0x6c815f 0x42bad1 # 0x5090c3 sync.(*Mutex).Lock+0xd3 /usr/local/go/src/sync/mutex.go:66 # 0x50a113 sync.(*RWMutex).Lock+0x33 /usr/local/go/src/sync/rwmutex.go:82 # 0x6c63f8 github.com/couchbase/goxdcr/parts.(*XmemNozzle).batchReady+0x68 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:860 # 0x6c815f github.com/couchbase/goxdcr/parts.(*XmemNozzle).processData_batch+0x80f /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:979 1 @ 0x418185 0x4181f8 0x4059a4 0x4051d3 0x6d8a47 0x6c761d 0x6c72cf 0x79d621 0x6b8ff0 0x42bad1 # 0x6d8a47 github.com/couchbase/goxdcr/parts.(*XmemNozzle).writeToDataChan+0x87 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2161 # 0x6c761d github.com/couchbase/goxdcr/parts.(*XmemNozzle).accumuBatch+0x9d /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:938 # 0x6c72cf github.com/couchbase/goxdcr/parts.(*XmemNozzle).Receive+0x9af /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:928 # 0x79d621 github.com/couchbase/goxdcr/connector.(*Router).Forward+0x221 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/connector/router.go:62 # 0x6b8ff0 github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0x1c80 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:431 3 @ 0x418185 0x41a42c 0x4198c2 0x67ed41 0x42bad1 # 0x67ed41 github.com/couchbase/goxdcr/pipeline_svc.(*TopologyChangeDetectorSvc).watch+0x211 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/topology_change_detector.go:89 1 @ 0x418185 0x41a42c 0x4198c2 0x56d2ce 0x42bad1 # 0x56d2ce github.com/couchbase/gomemcached/client.(*UprFeed).runFeed+0xa0e /home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/gomemcached/client/upr_feed.go:547 6 @ 0x418185 0x41a42c 0x4198c2 0x6bdcfd 0x42bad1 # 0x6bdcfd github.com/couchbase/goxdcr/parts.(*DcpNozzle).collectDcpDataChanLen+0x1ad /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:843 3 @ 0x418185 0x41a42c 0x4198c2 0x664e59 0x42bad1 # 0x664e59 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).checkpointing+0x419 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:633 2 @ 0x418185 0x41a42c 0x4198c2 0x66cb39 0x42bad1 # 0x66cb39 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).updateStats+0x549 /home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:273