goroutine profile: total 2324 1 @ 0x845558 0x845333 0x840c04 0xc6447e 0xc64690 0x67b20a 0x67ca7d 0x67d54e 0x679f6e 0x464201 # 0x845558 runtime/pprof.writeRuntimeProfile+0xb8 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/runtime/pprof/pprof.go:545 # 0x845333 runtime/pprof.writeGoroutine+0x93 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/runtime/pprof/pprof.go:507 # 0x840c04 runtime/pprof.(*Profile).WriteTo+0xd4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/runtime/pprof/pprof.go:236 # 0xc6447e net/http/pprof.handler.ServeHTTP+0x37e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/pprof/pprof.go:210 # 0xc64690 net/http/pprof.Index+0x200 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/pprof/pprof.go:222 # 0x67b20a net/http.HandlerFunc.ServeHTTP+0x3a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/server.go:1618 # 0x67ca7d net/http.(*ServeMux).ServeHTTP+0x17d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/server.go:1910 # 0x67d54e net/http.serverHandler.ServeHTTP+0x19e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/server.go:2081 # 0x679f6e net/http.(*conn).serve+0xf2e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/server.go:1472 1 @ 0x433283 0x441f97 0x4414f2 0x4f8702 0x4f6b88 0x4049f0 0x432ea0 0x464201 # 0x4f8702 github.com/couchbase/indexing/secondary/indexer.(*indexer).run+0x292 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:479 # 0x4f6b88 github.com/couchbase/indexing/secondary/indexer.NewIndexer+0x3b58 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:365 # 0x4049f0 main.main+0x14e0 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/cmd/indexer/main.go:105 # 0x432ea0 runtime.main+0x2b0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/runtime/proc.go:188 1 @ 0x464201 1 @ 0x433283 0x441f97 0x4414f2 0x78bb19 0x78b32d 0x78bdd7 0x5ccc4e 0x48e874 0x5ccd4d 0x464201 # 0x78bb19 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x7c9 /home/couchbase/jenkins/workspace/watson-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0x78b32d github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x6d /home/couchbase/jenkins/workspace/watson-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0x78bdd7 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x57 /home/couchbase/jenkins/workspace/watson-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x5ccc4e github.com/couchbase/indexing/secondary/indexer.NewSettingsManager.func1.1+0x1be /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/settings.go:70 # 0x48e874 github.com/couchbase/indexing/secondary/common.(*RetryHelper).Run+0x64 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/common/retry_helper.go:36 # 0x5ccd4d github.com/couchbase/indexing/secondary/indexer.NewSettingsManager.func1+0x9d /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/settings.go:74 1 @ 0x41362e 0x446682 0xc64e68 0x464201 # 0x446682 os/signal.signal_recv+0x132 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/runtime/sigqueue.go:116 # 0xc64e68 os/signal.loop+0x18 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/os/signal/signal_unix.go:22 1 @ 0x433283 0x42dd3e 0x42d200 0x6f667a 0x6f66e6 0x6f853a 0x70ca14 0xac0f39 0xac15b0 0x6d8df1 0x6c1607 0x6c1293 0x6c0b99 0xaf0e6c 0x6cb130 0x6cadbb 0x6ca25d 0x6d9816 0x6da257 0x47bf6a 0x47d95b 0x464201 # 0x42d200 net.runtime_pollWait+0x60 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/runtime/netpoll.go:160 # 0x6f667a net.(*pollDesc).Wait+0x3a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_poll_runtime.go:73 # 0x6f66e6 net.(*pollDesc).WaitRead+0x36 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_poll_runtime.go:78 # 0x6f853a net.(*netFD).Read+0x23a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_unix.go:250 # 0x70ca14 net.(*conn).Read+0xe4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/net.go:172 # 0xac0f39 bufio.(*Reader).fill+0x1e9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/bufio/bufio.go:97 # 0xac15b0 bufio.(*Reader).Read+0x260 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/bufio/bufio.go:207 # 0x6d8df1 github.com/couchbase/cbauth/revrpc.(*minirwc).Read+0x51 /home/couchbase/jenkins/workspace/watson-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:100 # 0x6c1607 encoding/json.(*Decoder).refill+0x287 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/encoding/json/stream.go:152 # 0x6c1293 encoding/json.(*Decoder).readValue+0x413 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/encoding/json/stream.go:128 # 0x6c0b99 encoding/json.(*Decoder).Decode+0x159 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/encoding/json/stream.go:57 # 0xaf0e6c net/rpc/jsonrpc.(*serverCodec).ReadRequestHeader+0x7c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/rpc/jsonrpc/server.go:66 # 0x6cb130 net/rpc.(*Server).readRequestHeader+0x90 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/rpc/server.go:576 # 0x6cadbb net/rpc.(*Server).readRequest+0x8b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/rpc/server.go:543 # 0x6ca25d net/rpc.(*Server).ServeCodec+0x7d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/rpc/server.go:462 # 0x6d9816 github.com/couchbase/cbauth/revrpc.(*Service).Run+0x9f6 /home/couchbase/jenkins/workspace/watson-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:146 # 0x6da257 github.com/couchbase/cbauth/revrpc.BabysitService+0x77 /home/couchbase/jenkins/workspace/watson-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:242 # 0x47bf6a github.com/couchbase/cbauth.runRPCForSvc+0x10a /home/couchbase/jenkins/workspace/watson-unix/godeps/src/github.com/couchbase/cbauth/default.go:56 # 0x47d95b github.com/couchbase/cbauth.startDefault.func1+0x2b /home/couchbase/jenkins/workspace/watson-unix/godeps/src/github.com/couchbase/cbauth/default.go:63 1 @ 0x433283 0x433344 0x44ea69 0x48735a 0x464201 # 0x44ea69 time.Sleep+0xf9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/runtime/time.go:59 # 0x48735a github.com/couchbase/indexing/secondary/common.pollForDeletedBuckets+0x3a /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/common/dcp_seqno.go:341 1 @ 0x629b05 0x6277bf 0x62663d 0x46f493 0x46c7ca 0x496b67 0x464201 # 0x629b05 syscall.Syscall+0x5 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/syscall/asm_linux_amd64.s:19 # 0x6277bf syscall.read+0x5f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/syscall/zsyscall_linux_amd64.go:783 # 0x62663d syscall.Read+0x4d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/syscall/syscall_unix.go:161 # 0x46f493 os.(*File).read+0x53 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/os/file_unix.go:228 # 0x46c7ca os.(*File).Read+0x8a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/os/file.go:95 # 0x496b67 github.com/couchbase/indexing/secondary/common.ExitOnStdinClose+0x97 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/common/util.go:318 7 @ 0x433283 0x42dd3e 0x42d200 0x6f667a 0x6f66e6 0x6f853a 0x70ca14 0x690467 0x6a5c90 0xac0f39 0xac115c 0x68c987 0x464201 # 0x42d200 net.runtime_pollWait+0x60 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/runtime/netpoll.go:160 # 0x6f667a net.(*pollDesc).Wait+0x3a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_poll_runtime.go:73 # 0x6f66e6 net.(*pollDesc).WaitRead+0x36 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_poll_runtime.go:78 # 0x6f853a net.(*netFD).Read+0x23a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_unix.go:250 # 0x70ca14 net.(*conn).Read+0xe4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/net.go:172 # 0x690467 net/http.noteEOFReader.Read+0x67 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/transport.go:1683 # 0x6a5c90 net/http.(*noteEOFReader).Read+0xd0 :284 # 0xac0f39 bufio.(*Reader).fill+0x1e9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/bufio/bufio.go:97 # 0xac115c bufio.(*Reader).Peek+0xcc /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/bufio/bufio.go:132 # 0x68c987 net/http.(*persistConn).readLoop+0x177 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/transport.go:1069 1 @ 0x433283 0x441f97 0x4414f2 0x4d2cc0 0x464201 # 0x4d2cc0 github.com/couchbase/indexing/secondary/indexer.(*compactionDaemon).loop+0x1610 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/compaction_manager.go:221 1 @ 0x433283 0x441f97 0x4414f2 0x8c1f62 0x464201 # 0x8c1f62 github.com/couchbase/indexing/secondary/manager/client.(*watcher).timeoutChecker+0x132 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/manager/client/metadata_provider.go:1326 1 @ 0x433283 0x441f97 0x4414f2 0xb7b0fd 0x464201 # 0xb7b0fd github.com/couchbase/gometa/protocol.(*Follower).startListener+0x36d /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/gometa/protocol/follower.go:149 3 @ 0x433283 0x42dd3e 0x42d200 0x6f667a 0x6f66e6 0x6f853a 0x70ca14 0xb62f8f 0xb62447 0x464201 # 0x42d200 net.runtime_pollWait+0x60 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/runtime/netpoll.go:160 # 0x6f667a net.(*pollDesc).Wait+0x3a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_poll_runtime.go:73 # 0x6f66e6 net.(*pollDesc).WaitRead+0x36 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_poll_runtime.go:78 # 0x6f853a net.(*netFD).Read+0x23a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_unix.go:250 # 0x70ca14 net.(*conn).Read+0xe4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/net.go:172 # 0xb62f8f github.com/couchbase/gometa/common.(*PeerPipe).readBytes+0xef /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/gometa/common/peerPipe.go:256 # 0xb62447 github.com/couchbase/gometa/common.(*PeerPipe).doReceive+0x67 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/gometa/common/peerPipe.go:201 3 @ 0x433283 0x433344 0x4096ef 0x40923b 0xb618fd 0x464201 # 0xb618fd github.com/couchbase/gometa/common.(*PeerPipe).doSend+0x8d /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/gometa/common/peerPipe.go:153 1 @ 0x433283 0x441f97 0x4414f2 0xb860bb 0x464201 # 0xb860bb github.com/couchbase/gometa/protocol.(*LeaderServer).listenFollower+0x68b /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/gometa/protocol/leaderServer.go:186 1 @ 0x433283 0x441f97 0x4414f2 0x546cde 0x464201 # 0x546cde github.com/couchbase/indexing/secondary/indexer.(*mutationMgr).handleWorkerMsgs+0x14e /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/mutation_manager.go:145 1 @ 0x433283 0x433344 0x4096ef 0x40923b 0xb813bd 0x464201 # 0xb813bd github.com/couchbase/gometa/protocol.(*Leader).listen+0xcd /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/gometa/protocol/leader.go:472 1 @ 0x433283 0x433344 0x44ea69 0x57b0d9 0x464201 # 0x44ea69 time.Sleep+0xf9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/runtime/time.go:59 # 0x57b0d9 github.com/couchbase/indexing/secondary/indexer.(*statsManager).runStatsDumpLogger+0x3f9 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/stats_manager.go:614 1 @ 0x433283 0x433344 0x4096ef 0x40923b 0x57a7f9 0x464201 # 0x57a7f9 github.com/couchbase/indexing/secondary/indexer.(*statsManager).run+0x59 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/stats_manager.go:570 1 @ 0x433283 0x433344 0x4096ef 0x40923b 0x4c9622 0x464201 # 0x4c9622 github.com/couchbase/indexing/secondary/indexer.(*clustMgrAgent).run+0xb2 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/cluster_manager_agent.go:121 1 @ 0x433283 0x441f97 0x4414f2 0x7c58b8 0x464201 # 0x7c58b8 github.com/couchbase/indexing/secondary/manager.(*IndexManager).monitorBucket+0x3c8 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/manager/manager.go:502 1 @ 0x433283 0x441f97 0x4414f2 0x7bba64 0x464201 # 0x7bba64 github.com/couchbase/indexing/secondary/manager.(*LifecycleMgr).processRequest+0x2d4 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/manager/lifecycle.go:162 1 @ 0x433283 0x441f97 0x4414f2 0xb87c1c 0xb85712 0xb9825d 0xb97845 0x464201 # 0xb87c1c github.com/couchbase/gometa/protocol.(*LeaderServer).processRequest+0x82c /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/gometa/protocol/leaderServer.go:384 # 0xb85712 github.com/couchbase/gometa/protocol.RunLeaderServerWithCustomHandler+0x652 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/gometa/protocol/leaderServer.go:130 # 0xb9825d github.com/couchbase/gometa/server.(*EmbeddedServer).runOnce+0x19d /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/gometa/server/embeddedServer.go:451 # 0xb97845 github.com/couchbase/gometa/server.(*EmbeddedServer).run+0x25 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/gometa/server/embeddedServer.go:354 1 @ 0x433283 0x42dd3e 0x42d200 0x6f667a 0x6f66e6 0x6fa37c 0x718a4d 0x718c8d 0xb5e7ab 0x464201 # 0x42d200 net.runtime_pollWait+0x60 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/runtime/netpoll.go:160 # 0x6f667a net.(*pollDesc).Wait+0x3a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_poll_runtime.go:73 # 0x6f66e6 net.(*pollDesc).WaitRead+0x36 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_poll_runtime.go:78 # 0x6fa37c net.(*netFD).accept+0x27c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_unix.go:426 # 0x718a4d net.(*TCPListener).AcceptTCP+0x4d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/tcpsock_posix.go:254 # 0x718c8d net.(*TCPListener).Accept+0x3d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/tcpsock_posix.go:264 # 0xb5e7ab github.com/couchbase/gometa/common.(*PeerListener).listen+0x7b /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/gometa/common/peerListener.go:135 1 @ 0x433283 0x433344 0x4096ef 0x40920b 0x8a2ace 0x464201 # 0x8a2ace github.com/couchbase/indexing/secondary/queryport/client.(*metadataClient).logstats+0x9e /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/queryport/client/meta_client.go:412 4 @ 0x433283 0x42dd3e 0x42d200 0x6f667a 0x6f66e6 0x6f853a 0x70ca14 0x673776 0xac0f39 0xac1daa 0xac1e23 0xabd0f1 0xabce60 0x66cd86 0x674f79 0x679987 0x464201 # 0x42d200 net.runtime_pollWait+0x60 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/runtime/netpoll.go:160 # 0x6f667a net.(*pollDesc).Wait+0x3a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_poll_runtime.go:73 # 0x6f66e6 net.(*pollDesc).WaitRead+0x36 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_poll_runtime.go:78 # 0x6f853a net.(*netFD).Read+0x23a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_unix.go:250 # 0x70ca14 net.(*conn).Read+0xe4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/net.go:172 # 0x673776 net/http.(*connReader).Read+0x196 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/server.go:526 # 0xac0f39 bufio.(*Reader).fill+0x1e9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/bufio/bufio.go:97 # 0xac1daa bufio.(*Reader).ReadSlice+0x21a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/bufio/bufio.go:328 # 0xac1e23 bufio.(*Reader).ReadLine+0x53 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/bufio/bufio.go:357 # 0xabd0f1 net/textproto.(*Reader).readLineSlice+0x81 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/textproto/reader.go:55 # 0xabce60 net/textproto.(*Reader).ReadLine+0x40 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/textproto/reader.go:36 # 0x66cd86 net/http.readRequest+0xb6 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/request.go:721 # 0x674f79 net/http.(*conn).readRequest+0x359 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/server.go:705 # 0x679987 net/http.(*conn).serve+0x947 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/server.go:1425 1030 @ 0x433283 0x42dd3e 0x42d200 0x6f667a 0x6f66e6 0x6f853a 0x70ca14 0xba0b79 0xba17d6 0xba02a0 0xa6dc9d 0x464201 # 0x42d200 net.runtime_pollWait+0x60 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/runtime/netpoll.go:160 # 0x6f667a net.(*pollDesc).Wait+0x3a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_poll_runtime.go:73 # 0x6f66e6 net.(*pollDesc).WaitRead+0x36 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_poll_runtime.go:78 # 0x6f853a net.(*netFD).Read+0x23a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_unix.go:250 # 0x70ca14 net.(*conn).Read+0xe4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/net.go:172 # 0xba0b79 github.com/couchbase/indexing/secondary/transport.fullRead+0xa9 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/transport/transport.go:188 # 0xba17d6 github.com/couchbase/indexing/secondary/transport.Receive+0x116 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/transport/util.go:46 # 0xba02a0 github.com/couchbase/indexing/secondary/transport.(*TransportPacket).Receive+0xb0 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/transport/transport.go:116 # 0xa6dc9d github.com/couchbase/indexing/secondary/queryport.(*Server).doReceive+0x44d /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/queryport/server.go:173 1 @ 0x433283 0x441f97 0x4414f2 0xb89eb4 0xb88e76 0xb888c6 0x464201 # 0xb89eb4 github.com/couchbase/gometa/protocol.runWatcher+0x654 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/gometa/protocol/watcherServer.go:320 # 0xb88e76 github.com/couchbase/gometa/protocol.runOnce+0x466 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/gometa/protocol/watcherServer.go:176 # 0xb888c6 github.com/couchbase/gometa/protocol.RunWatcherServerWithRequest+0x146 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/gometa/protocol/watcherServer.go:49 1 @ 0x433283 0x433344 0x4096ef 0x40923b 0x546a88 0x464201 # 0x546a88 github.com/couchbase/indexing/secondary/indexer.(*mutationMgr).run+0xc8 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/mutation_manager.go:121 1 @ 0x433283 0x42dd3e 0x42d200 0x6f667a 0x6f66e6 0x6f853a 0x70ca14 0x690467 0x6a5c90 0xac0f39 0xac1daa 0xad0ce3 0xad05c9 0xad0877 0x684cd1 0x684bc0 0x68fd5a 0x6c1607 0x6c1293 0x6c0b99 0x78ebb4 0x464201 # 0x42d200 net.runtime_pollWait+0x60 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/runtime/netpoll.go:160 # 0x6f667a net.(*pollDesc).Wait+0x3a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_poll_runtime.go:73 # 0x6f66e6 net.(*pollDesc).WaitRead+0x36 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_poll_runtime.go:78 # 0x6f853a net.(*netFD).Read+0x23a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_unix.go:250 # 0x70ca14 net.(*conn).Read+0xe4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/net.go:172 # 0x690467 net/http.noteEOFReader.Read+0x67 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/transport.go:1683 # 0x6a5c90 net/http.(*noteEOFReader).Read+0xd0 :284 # 0xac0f39 bufio.(*Reader).fill+0x1e9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/bufio/bufio.go:97 # 0xac1daa bufio.(*Reader).ReadSlice+0x21a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/bufio/bufio.go:328 # 0xad0ce3 net/http/internal.readChunkLine+0x53 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/internal/chunked.go:110 # 0xad05c9 net/http/internal.(*chunkedReader).beginChunk+0x39 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/internal/chunked.go:47 # 0xad0877 net/http/internal.(*chunkedReader).Read+0xb7 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/internal/chunked.go:77 # 0x684cd1 net/http.(*body).readLocked+0xa1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/transfer.go:651 # 0x684bc0 net/http.(*body).Read+0x110 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/transfer.go:643 # 0x68fd5a net/http.(*bodyEOFSignal).Read+0x26a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/transport.go:1609 # 0x6c1607 encoding/json.(*Decoder).refill+0x287 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/encoding/json/stream.go:152 # 0x6c1293 encoding/json.(*Decoder).readValue+0x413 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/encoding/json/stream.go:128 # 0x6c0b99 encoding/json.(*Decoder).Decode+0x159 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/encoding/json/stream.go:57 # 0x78ebb4 github.com/couchbase/cbauth/metakv.doRunObserveChildren.func1+0x114 /home/couchbase/jenkins/workspace/watson-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:270 10 @ 0x433283 0x441f97 0x4414f2 0x68e3d2 0x464201 # 0x68e3d2 net/http.(*persistConn).writeLoop+0x472 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/transport.go:1273 3 @ 0x433283 0x441f97 0x4414f2 0x68d562 0x464201 # 0x68d562 net/http.(*persistConn).readLoop+0xd52 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/transport.go:1178 1 @ 0x433283 0x441f97 0x4414f2 0x547210 0x464201 # 0x547210 github.com/couchbase/indexing/secondary/indexer.(*mutationMgr).listenWorkerMsgs+0x520 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/mutation_manager.go:168 1 @ 0x433283 0x433344 0x4096ef 0x40923b 0x522828 0x464201 # 0x522828 github.com/couchbase/indexing/secondary/indexer.(*kvSender).run+0x58 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/kv_sender.go:85 1 @ 0x433283 0x433344 0x4096ef 0x40923b 0x599ac8 0x464201 # 0x599ac8 github.com/couchbase/indexing/secondary/indexer.(*timekeeper).run+0x58 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/timekeeper.go:111 1 @ 0x433283 0x42dd3e 0x42d200 0x6f667a 0x6f66e6 0x6fa37c 0x718a4d 0x718c8d 0xa6d44d 0x464201 # 0x42d200 net.runtime_pollWait+0x60 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/runtime/netpoll.go:160 # 0x6f667a net.(*pollDesc).Wait+0x3a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_poll_runtime.go:73 # 0x6f66e6 net.(*pollDesc).WaitRead+0x36 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_poll_runtime.go:78 # 0x6fa37c net.(*netFD).accept+0x27c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_unix.go:426 # 0x718a4d net.(*TCPListener).AcceptTCP+0x4d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/tcpsock_posix.go:254 # 0x718c8d net.(*TCPListener).Accept+0x3d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/tcpsock_posix.go:264 # 0xa6d44d github.com/couchbase/indexing/secondary/queryport.(*Server).listener+0x6d /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/queryport/server.go:118 1 @ 0x433283 0x433344 0x4096ef 0x40923b 0x5607b8 0x464201 # 0x5607b8 github.com/couchbase/indexing/secondary/indexer.(*scanCoordinator).run+0x58 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/scan_coordinator.go:318 1 @ 0x433283 0x433344 0x4096ef 0x40923b 0x5604d5 0x464201 # 0x5604d5 github.com/couchbase/indexing/secondary/indexer.(*scanCoordinator).listenSnapshot+0x55 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/scan_coordinator.go:267 1 @ 0x433283 0x433344 0x4096ef 0x40923b 0x4d2f4d 0x464201 # 0x4d2f4d github.com/couchbase/indexing/secondary/indexer.(*compactionManager).run+0x9d /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/compaction_manager.go:314 1 @ 0x433283 0x42dd3e 0x42d200 0x6f667a 0x6f66e6 0x6f853a 0x70ca14 0x690467 0x6a5c90 0xac0f39 0xac1daa 0xad0ce3 0xad05c9 0xad0877 0x684cd1 0x684bc0 0x68fd5a 0xac0f39 0xac1daa 0xac20e9 0x73ce50 0x73c51b 0x48eaa6 0x464201 # 0x42d200 net.runtime_pollWait+0x60 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/runtime/netpoll.go:160 # 0x6f667a net.(*pollDesc).Wait+0x3a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_poll_runtime.go:73 # 0x6f66e6 net.(*pollDesc).WaitRead+0x36 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_poll_runtime.go:78 # 0x6f853a net.(*netFD).Read+0x23a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_unix.go:250 # 0x70ca14 net.(*conn).Read+0xe4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/net.go:172 # 0x690467 net/http.noteEOFReader.Read+0x67 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/transport.go:1683 # 0x6a5c90 net/http.(*noteEOFReader).Read+0xd0 :284 # 0xac0f39 bufio.(*Reader).fill+0x1e9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/bufio/bufio.go:97 # 0xac1daa bufio.(*Reader).ReadSlice+0x21a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/bufio/bufio.go:328 # 0xad0ce3 net/http/internal.readChunkLine+0x53 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/internal/chunked.go:110 # 0xad05c9 net/http/internal.(*chunkedReader).beginChunk+0x39 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/internal/chunked.go:47 # 0xad0877 net/http/internal.(*chunkedReader).Read+0xb7 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/internal/chunked.go:77 # 0x684cd1 net/http.(*body).readLocked+0xa1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/transfer.go:651 # 0x684bc0 net/http.(*body).Read+0x110 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/transfer.go:643 # 0x68fd5a net/http.(*bodyEOFSignal).Read+0x26a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/transport.go:1609 # 0xac0f39 bufio.(*Reader).fill+0x1e9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/bufio/bufio.go:97 # 0xac1daa bufio.(*Reader).ReadSlice+0x21a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/bufio/bufio.go:328 # 0xac20e9 bufio.(*Reader).ReadBytes+0xa9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/bufio/bufio.go:406 # 0x73ce50 github.com/couchbase/indexing/secondary/dcp.(*Client).runObserveStreamingEndpoint+0x810 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/dcp/pools.go:366 # 0x73c51b github.com/couchbase/indexing/secondary/dcp.(*Client).RunObservePool+0xab /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/dcp/pools.go:305 # 0x48eaa6 github.com/couchbase/indexing/secondary/common.(*serviceNotifierInstance).RunPoolObserver+0xa6 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/common/services_notifier.go:78 1 @ 0x433283 0x42dd3e 0x42d200 0x6f667a 0x6f66e6 0x6f853a 0x70ca14 0x690467 0x6a5c90 0xac0f39 0xac1daa 0xad0ce3 0xad05c9 0xad0877 0x684cd1 0x684bc0 0x68fd5a 0xac0f39 0xac1daa 0xac20e9 0x73ce50 0x73c613 0x48eda6 0x464201 # 0x42d200 net.runtime_pollWait+0x60 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/runtime/netpoll.go:160 # 0x6f667a net.(*pollDesc).Wait+0x3a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_poll_runtime.go:73 # 0x6f66e6 net.(*pollDesc).WaitRead+0x36 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_poll_runtime.go:78 # 0x6f853a net.(*netFD).Read+0x23a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_unix.go:250 # 0x70ca14 net.(*conn).Read+0xe4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/net.go:172 # 0x690467 net/http.noteEOFReader.Read+0x67 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/transport.go:1683 # 0x6a5c90 net/http.(*noteEOFReader).Read+0xd0 :284 # 0xac0f39 bufio.(*Reader).fill+0x1e9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/bufio/bufio.go:97 # 0xac1daa bufio.(*Reader).ReadSlice+0x21a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/bufio/bufio.go:328 # 0xad0ce3 net/http/internal.readChunkLine+0x53 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/internal/chunked.go:110 # 0xad05c9 net/http/internal.(*chunkedReader).beginChunk+0x39 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/internal/chunked.go:47 # 0xad0877 net/http/internal.(*chunkedReader).Read+0xb7 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/internal/chunked.go:77 # 0x684cd1 net/http.(*body).readLocked+0xa1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/transfer.go:651 # 0x684bc0 net/http.(*body).Read+0x110 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/transfer.go:643 # 0x68fd5a net/http.(*bodyEOFSignal).Read+0x26a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/transport.go:1609 # 0xac0f39 bufio.(*Reader).fill+0x1e9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/bufio/bufio.go:97 # 0xac1daa bufio.(*Reader).ReadSlice+0x21a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/bufio/bufio.go:328 # 0xac20e9 bufio.(*Reader).ReadBytes+0xa9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/bufio/bufio.go:406 # 0x73ce50 github.com/couchbase/indexing/secondary/dcp.(*Client).runObserveStreamingEndpoint+0x810 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/dcp/pools.go:366 # 0x73c613 github.com/couchbase/indexing/secondary/dcp.(*Client).RunObserveNodeServices+0xc3 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/dcp/pools.go:318 # 0x48eda6 github.com/couchbase/indexing/secondary/common.(*serviceNotifierInstance).RunServicesObserver+0xa6 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/common/services_notifier.go:87 48 @ 0x433283 0x433344 0x4096ef 0x40923b 0x82f1f2 0x464201 # 0x82f1f2 github.com/couchbase/indexing/secondary/memdb.(*MemDB).freeWorker+0x52 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/memdb/memdb.go:626 1 @ 0x433283 0x433344 0x4096ef 0x40920b 0x5e0709 0x464201 # 0x5e0709 github.com/couchbase/indexing/secondary/logging.PeriodicProfile.func1+0x59 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/logging/logging.go:196 1 @ 0x433283 0x42dd3e 0x42d200 0x6f667a 0x6f66e6 0x6fa37c 0x718a4d 0x67e701 0x67d899 0x67d6e6 0x67dde8 0x5bca11 0x464201 # 0x42d200 net.runtime_pollWait+0x60 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/runtime/netpoll.go:160 # 0x6f667a net.(*pollDesc).Wait+0x3a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_poll_runtime.go:73 # 0x6f66e6 net.(*pollDesc).WaitRead+0x36 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_poll_runtime.go:78 # 0x6fa37c net.(*netFD).accept+0x27c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_unix.go:426 # 0x718a4d net.(*TCPListener).AcceptTCP+0x4d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/tcpsock_posix.go:254 # 0x67e701 net/http.tcpKeepAliveListener.Accept+0x41 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/server.go:2427 # 0x67d899 net/http.(*Server).Serve+0x129 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/server.go:2117 # 0x67d6e6 net/http.(*Server).ListenAndServe+0x136 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/server.go:2098 # 0x67dde8 net/http.ListenAndServe+0x98 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/http/server.go:2195 # 0x5bca11 github.com/couchbase/indexing/secondary/indexer.NewIndexer.func1+0x41 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:268 1 @ 0x433283 0x441f97 0x4414f2 0x8a5f6e 0x464201 # 0x8a5f6e github.com/couchbase/indexing/secondary/queryport/client.(*metadataClient).watchClusterChanges+0x52e /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/queryport/client/meta_client.go:774 2 @ 0x433283 0x441f97 0x4414f2 0xb81143 0x464201 # 0xb81143 github.com/couchbase/gometa/protocol.(*messageListener).start+0x5a3 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/gometa/protocol/leader.go:403 1 @ 0x433283 0x433344 0x4096ef 0x40923b 0x4c3a48 0x464201 # 0x4c3a48 github.com/couchbase/indexing/secondary/indexer.(*adminMgr).run+0x58 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/admin_manager.go:51 1 @ 0x433283 0x433344 0x4096ef 0x40923b 0x57ba08 0x464201 # 0x57ba08 github.com/couchbase/indexing/secondary/indexer.(*storageMgr).run+0x58 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/storage_manager.go:144 44 @ 0x433283 0x433344 0x442e17 0x442a66 0x62a644 0x88b662 0x82c881 0x535d94 0x5350ac 0x53494e 0x464201 # 0x442a66 sync.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/runtime/sema.go:47 # 0x62a644 sync.(*Mutex).Lock+0x1c4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/sync/mutex.go:83 # 0x88b662 github.com/couchbase/indexing/secondary/memdb/skiplist.(*AccessBarrier).FlushSession+0x42 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/memdb/skiplist/access_barrier.go:165 # 0x82c881 github.com/couchbase/indexing/secondary/memdb.(*Writer).DeleteNode+0x101 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/memdb/memdb.go:246 # 0x535d94 github.com/couchbase/indexing/secondary/indexer.(*memdbSlice).insertSecIndex+0x764 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/memdb_slice_impl.go:381 # 0x5350ac github.com/couchbase/indexing/secondary/indexer.(*memdbSlice).insert+0x14c /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/memdb_slice_impl.go:338 # 0x53494e github.com/couchbase/indexing/secondary/indexer.(*memdbSlice).handleCommandsWorker+0x32e /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/memdb_slice_impl.go:298 1 @ 0x433283 0x433344 0x44ea69 0x51f6cf 0x464201 # 0x44ea69 time.Sleep+0xf9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/runtime/time.go:59 # 0x51f6cf github.com/couchbase/indexing/secondary/indexer.(*indexer).monitorMemUsage+0x97f /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:4049 1 @ 0x433283 0x433344 0x44ea69 0x521d3b 0x464201 # 0x44ea69 time.Sleep+0xf9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/runtime/time.go:59 # 0x521d3b github.com/couchbase/indexing/secondary/indexer.(*indexer).logMemstats+0x1eb /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:4296 1 @ 0x433283 0x441f97 0x4414f2 0x4f8ff1 0x464201 # 0x4f8ff1 github.com/couchbase/indexing/secondary/indexer.(*indexer).listenWorkerMsgs+0x291 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:572 1 @ 0x433283 0x441f97 0x4414f2 0x4f8a22 0x464201 # 0x4f8a22 github.com/couchbase/indexing/secondary/indexer.(*indexer).listenAdminMsgs+0x312 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:513 1030 @ 0x433283 0x441f97 0x4414f2 0x562cfb 0x56486f 0x5c8b92 0xa6d7c3 0x464201 # 0x562cfb github.com/couchbase/indexing/secondary/indexer.(*scanCoordinator).getRequestedIndexSnapshot+0x4bb /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/scan_coordinator.go:633 # 0x56486f github.com/couchbase/indexing/secondary/indexer.(*scanCoordinator).serverCallback+0x5cf /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/scan_coordinator.go:774 # 0x5c8b92 github.com/couchbase/indexing/secondary/indexer.(*scanCoordinator).(github.com/couchbase/indexing/secondary/indexer.serverCallback)-fm+0x52 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/scan_coordinator.go:244 # 0xa6d7c3 github.com/couchbase/indexing/secondary/queryport.(*Server).handleConnection+0x223 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/queryport/server.go:148 32 @ 0x433283 0x441f97 0x4414f2 0x5871b7 0x464201 # 0x5871b7 github.com/couchbase/indexing/secondary/indexer.(*streamWorker).start+0x247 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/stream_reader.go:476 1 @ 0x433283 0x441f97 0x4414f2 0x534f38 0x464201 # 0x534f38 github.com/couchbase/indexing/secondary/indexer.(*memdbSlice).handleCommandsWorker+0x918 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/memdb_slice_impl.go:293 48 @ 0x433283 0x441f97 0x4414f2 0x82f153 0x464201 # 0x82f153 github.com/couchbase/indexing/secondary/memdb.(*MemDB).collectionWorker+0x393 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/memdb/memdb.go:604 2 @ 0x88d850 0x88e53e 0x88ebc6 0x88e904 0x88e85e 0x88b53a 0x88ef14 0x88e904 0x82c499 0x535b65 0x5350ac 0x53494e 0x464201 # 0x88d850 github.com/couchbase/indexing/secondary/memdb/skiplist.(*Node).getNext+0x0 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/memdb/skiplist/node_amd64.go:70 # 0x88e53e github.com/couchbase/indexing/secondary/memdb/skiplist.(*Skiplist).findPath+0x7e /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/memdb/skiplist/skiplist.go:168 # 0x88ebc6 github.com/couchbase/indexing/secondary/memdb/skiplist.(*Skiplist).Insert3+0x296 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/memdb/skiplist/skiplist.go:225 # 0x88e904 github.com/couchbase/indexing/secondary/memdb/skiplist.(*Skiplist).Insert2+0x84 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/memdb/skiplist/skiplist.go:210 # 0x88e85e github.com/couchbase/indexing/secondary/memdb/skiplist.(*Skiplist).Insert+0x5e /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/memdb/skiplist/skiplist.go:203 # 0x88b53a github.com/couchbase/indexing/secondary/memdb/skiplist.(*AccessBarrier).Release+0x1da /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/memdb/skiplist/access_barrier.go:153 # 0x88ef14 github.com/couchbase/indexing/secondary/memdb/skiplist.(*Skiplist).Insert3+0x5e4 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/memdb/skiplist/skiplist.go:253 # 0x88e904 github.com/couchbase/indexing/secondary/memdb/skiplist.(*Skiplist).Insert2+0x84 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/memdb/skiplist/skiplist.go:210 # 0x82c499 github.com/couchbase/indexing/secondary/memdb.(*Writer).Put2+0x109 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/memdb/memdb.go:196 # 0x535b65 github.com/couchbase/indexing/secondary/indexer.(*memdbSlice).insertSecIndex+0x535 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/memdb_slice_impl.go:373 # 0x5350ac github.com/couchbase/indexing/secondary/indexer.(*memdbSlice).insert+0x14c /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/memdb_slice_impl.go:338 # 0x53494e github.com/couchbase/indexing/secondary/indexer.(*memdbSlice).handleCommandsWorker+0x32e /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/memdb_slice_impl.go:298 1 @ 0x88c880 0x88e685 0x88ebc6 0x88e904 0x88e85e 0x88b53a 0x88b7ba 0x82c881 0x535d94 0x5350ac 0x53494e 0x464201 # 0x88c880 github.com/couchbase/indexing/secondary/memdb/skiplist.compare+0x0 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/memdb/skiplist/item.go:9 # 0x88e685 github.com/couchbase/indexing/secondary/memdb/skiplist.(*Skiplist).findPath+0x1c5 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/memdb/skiplist/skiplist.go:182 # 0x88ebc6 github.com/couchbase/indexing/secondary/memdb/skiplist.(*Skiplist).Insert3+0x296 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/memdb/skiplist/skiplist.go:225 # 0x88e904 github.com/couchbase/indexing/secondary/memdb/skiplist.(*Skiplist).Insert2+0x84 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/memdb/skiplist/skiplist.go:210 # 0x88e85e github.com/couchbase/indexing/secondary/memdb/skiplist.(*Skiplist).Insert+0x5e /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/memdb/skiplist/skiplist.go:203 # 0x88b53a github.com/couchbase/indexing/secondary/memdb/skiplist.(*AccessBarrier).Release+0x1da /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/memdb/skiplist/access_barrier.go:153 # 0x88b7ba github.com/couchbase/indexing/secondary/memdb/skiplist.(*AccessBarrier).FlushSession+0x19a /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/memdb/skiplist/access_barrier.go:177 # 0x82c881 github.com/couchbase/indexing/secondary/memdb.(*Writer).DeleteNode+0x101 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/memdb/memdb.go:246 # 0x535d94 github.com/couchbase/indexing/secondary/indexer.(*memdbSlice).insertSecIndex+0x764 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/memdb_slice_impl.go:381 # 0x5350ac github.com/couchbase/indexing/secondary/indexer.(*memdbSlice).insert+0x14c /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/memdb_slice_impl.go:338 # 0x53494e github.com/couchbase/indexing/secondary/indexer.(*memdbSlice).handleCommandsWorker+0x32e /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/memdb_slice_impl.go:298 1 @ 0x433283 0x433344 0x4096ef 0x40923b 0x485148 0x464201 # 0x485148 github.com/couchbase/indexing/secondary/common.(*vbSeqnosReader).Routine+0x68 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/common/dcp_seqno.go:104 1 @ 0x433283 0x441f97 0x4414f2 0xa7527a 0x464201 # 0xa7527a github.com/couchbase/indexing/secondary/dataport.(*Server).genServer+0xaea /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/dataport/server.go:338 1 @ 0x433283 0x441f97 0x4414f2 0x5d3e3d 0x464201 # 0x5d3e3d github.com/couchbase/indexing/secondary/indexer.(*timekeeper).startTimer.func1+0x11d /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/timekeeper.go:2848 1 @ 0x433283 0x42dd3e 0x42d200 0x6f667a 0x6f66e6 0x6fa37c 0x718a4d 0x718c8d 0xa775dc 0x464201 # 0x42d200 net.runtime_pollWait+0x60 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/runtime/netpoll.go:160 # 0x6f667a net.(*pollDesc).Wait+0x3a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_poll_runtime.go:73 # 0x6f66e6 net.(*pollDesc).WaitRead+0x36 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_poll_runtime.go:78 # 0x6fa37c net.(*netFD).accept+0x27c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_unix.go:426 # 0x718a4d net.(*TCPListener).AcceptTCP+0x4d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/tcpsock_posix.go:254 # 0x718c8d net.(*TCPListener).Accept+0x3d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/tcpsock_posix.go:264 # 0xa775dc github.com/couchbase/indexing/secondary/dataport.listener+0x3c /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/dataport/server.go:515 1 @ 0x433283 0x441f97 0x4414f2 0x583a90 0x464201 # 0x583a90 github.com/couchbase/indexing/secondary/indexer.(*mutationStreamReader).run+0x430 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/stream_reader.go:157 1 @ 0x433283 0x433344 0x4096ef 0x40923b 0x583b26 0x464201 # 0x583b26 github.com/couchbase/indexing/secondary/indexer.(*mutationStreamReader).listenSupvCmd+0x76 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/stream_reader.go:198 1 @ 0x433283 0x441f97 0x4414f2 0x585543 0x464201 # 0x585543 github.com/couchbase/indexing/secondary/indexer.(*mutationStreamReader).syncWorker+0x123 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/stream_reader.go:347 4 @ 0x433283 0x42dd3e 0x42d200 0x6f667a 0x6f66e6 0x6f853a 0x70ca14 0xba0b79 0xba17d6 0xba02a0 0xa77dde 0x464201 # 0x42d200 net.runtime_pollWait+0x60 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/runtime/netpoll.go:160 # 0x6f667a net.(*pollDesc).Wait+0x3a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_poll_runtime.go:73 # 0x6f66e6 net.(*pollDesc).WaitRead+0x36 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_poll_runtime.go:78 # 0x6f853a net.(*netFD).Read+0x23a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/fd_unix.go:250 # 0x70ca14 net.(*conn).Read+0xe4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/net/net.go:172 # 0xba0b79 github.com/couchbase/indexing/secondary/transport.fullRead+0xa9 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/transport/transport.go:188 # 0xba17d6 github.com/couchbase/indexing/secondary/transport.Receive+0x116 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/transport/util.go:46 # 0xba02a0 github.com/couchbase/indexing/secondary/transport.(*TransportPacket).Receive+0xb0 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/transport/transport.go:116 # 0xa77dde github.com/couchbase/indexing/secondary/dataport.doReceive+0x2fe /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/dataport/server.go:563 1 @ 0x433283 0x433344 0x442e17 0x442a66 0x62bca4 0x57c7ce 0x464201 # 0x442a66 sync.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/runtime/sema.go:47 # 0x62bca4 sync.(*WaitGroup).Wait+0xb4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/sync/waitgroup.go:127 # 0x57c7ce github.com/couchbase/indexing/secondary/indexer.(*storageMgr).createSnapshotWorker+0x25e /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/storage_manager.go:390 1 @ 0x433283 0x433344 0x44ea69 0x53bfed 0x53cc81 0x5ce15f 0x464201 # 0x44ea69 time.Sleep+0xf9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.6/go/src/runtime/time.go:59 # 0x53bfed github.com/couchbase/indexing/secondary/indexer.(*memdbSlice).waitPersist+0x18d /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/memdb_slice_impl.go:856 # 0x53cc81 github.com/couchbase/indexing/secondary/indexer.(*memdbSlice).IsDirty+0x21 /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/memdb_slice_impl.go:988 # 0x5ce15f github.com/couchbase/indexing/secondary/indexer.(*storageMgr).createSnapshotWorker.func1+0x71f /home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/storage_manager.go:302