Indexer panics creating index in Spock

I downloaded Spock, build 3518.

I created a bucket that has about 1 million documents, total size around 1GB.

I created a primary index on the bucket. It seemed to work fine. 

I created an index on one of the fields. The indexer then gets stuck in a cycle and panic/restart.

I'm attaching indexer.log, which is full of panic messages such as:

goroutine 53 [select]:

runtime.gopark(0x4b29a10, 0x0, 0x4a67d1a, 0x6, 0x18, 0x2)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:259 +0x13a fp=0xc4201b1c88 sp=0xc4201b1c58

runtime.selectgoImpl(0xc4201b1f28, 0xc4201b1f06, 0x20)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/select.go:423 +0x11d9 fp=0xc4201b1eb0 sp=0xc4201b1c88

runtime.selectgo(0xc4201b1f28)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/select.go:238 +0x1c fp=0xc4201b1ed8 sp=0xc4201b1eb0

github.com/couchbase/indexing/secondary/indexer.(*mutationMgr).handleWorkerMsgs(0xc4200ee000)

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/mutation_manager.go:146 +0x162 fp=0xc4201b1fb8 sp=0xc4201b1ed8

runtime.goexit()

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc4201b1fc0 sp=0xc4201b1fb8

created by github.com/couchbase/indexing/secondary/indexer.(*mutationMgr).run

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/mutation_manager.go:115 +0x6d

 

goroutine 54 [select]:

runtime.gopark(0x4b29a10, 0x0, 0x4a67d1a, 0x6, 0x18, 0x2)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:259 +0x13a fp=0xc423049bc0 sp=0xc423049b90

runtime.selectgoImpl(0xc423049f18, 0xc4201b2648, 0x20)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/select.go:423 +0x11d9 fp=0xc423049de8 sp=0xc423049bc0

runtime.selectgo(0xc423049f18)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/select.go:238 +0x1c fp=0xc423049e10 sp=0xc423049de8

github.com/couchbase/indexing/secondary/indexer.(*mutationMgr).listenWorkerMsgs(0xc4200ee000)

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/mutation_manager.go:169 +0x497 fp=0xc423049fa8 sp=0xc423049e10

runtime.goexit()

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc423049fb0 sp=0xc423049fa8

created by github.com/couchbase/indexing/secondary/indexer.(*mutationMgr).run

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/mutation_manager.go:116 +0x8f

 

goroutine 25 [chan receive]:

runtime.gopark(0x4b299d0, 0xc421190658, 0x4a6d636, 0xc, 0xc421174e17, 0x3)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:259 +0x13a fp=0xc421174e50 sp=0xc421174e20

runtime.goparkunlock(0xc421190658, 0x4a6d636, 0xc, 0x50aa817, 0x3)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:265 +0x5e fp=0xc421174e90 sp=0xc421174e50

runtime.chanrecv(0x4926820, 0xc421190600, 0xc421174f98, 0xc421174f01, 0xc421170101)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/chan.go:496 +0x2df fp=0xc421174f18 sp=0xc421174e90

runtime.chanrecv2(0x4926820, 0xc421190600, 0xc421174f98, 0x1)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/chan.go:383 +0x35 fp=0xc421174f50 sp=0xc421174f18

github.com/couchbase/indexing/secondary/indexer.(*scanCoordinator).run(0xc4201da000)

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/scan_coordinator.go:508 +0x63 fp=0xc421174fb8 sp=0xc421174f50

runtime.goexit()

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc421174fc0 sp=0xc421174fb8

created by github.com/couchbase/indexing/secondary/indexer.NewScanCoordinator

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/scan_coordinator.go:438 +0x42b

 

goroutine 26 [chan receive]:

runtime.gopark(0x4b299d0, 0xc421190718, 0x4a6d636, 0xc, 0x17, 0x3)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:259 +0x13a fp=0xc421174670 sp=0xc421174640

runtime.goparkunlock(0xc421190718, 0x4a6d636, 0xc, 0x17, 0x3)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:265 +0x5e fp=0xc4211746b0 sp=0xc421174670

runtime.chanrecv(0x48ed560, 0xc4211906c0, 0xc421174798, 0x1, 0x0)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/chan.go:496 +0x2df fp=0xc421174738 sp=0xc4211746b0

runtime.chanrecv2(0x48ed560, 0xc4211906c0, 0xc421174798, 0x0)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/chan.go:383 +0x35 fp=0xc421174770 sp=0xc421174738

github.com/couchbase/indexing/secondary/indexer.(*scanCoordinator).listenSnapshot(0xc4201da000)

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/scan_coordinator.go:446 +0x60 fp=0xc4211747b8 sp=0xc421174770

runtime.goexit()

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc4211747c0 sp=0xc4211747b8

created by github.com/couchbase/indexing/secondary/indexer.NewScanCoordinator

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/scan_coordinator.go:439 +0x44d

 

goroutine 16 [select]:

runtime.gopark(0x4b29a10, 0x0, 0x4a67d1a, 0x6, 0x18, 0x2)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:259 +0x13a fp=0xc420156c20 sp=0xc420156bf0

runtime.selectgoImpl(0xc420156f10, 0x0, 0x18)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/select.go:423 +0x11d9 fp=0xc420156e48 sp=0xc420156c20

runtime.selectgo(0xc420156f10)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/select.go:238 +0x1c fp=0xc420156e70 sp=0xc420156e48

github.com/couchbase/gometa/protocol.(*LeaderServer).listenFollower(0xc4220deb80, 0xc4230953b0)

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gometa/protocol/leaderServer.go:186 +0x4b6 fp=0xc420156fa0 sp=0xc420156e70

runtime.goexit()

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc420156fa8 sp=0xc420156fa0

created by github.com/couchbase/gometa/protocol.RunLeaderServerWithCustomHandler

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gometa/protocol/leaderServer.go:125 +0x58a

 

goroutine 15 [chan receive]:

runtime.gopark(0x4b299d0, 0xc42000eef8, 0x4a6d636, 0xc, 0xc42013a717, 0x3)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:259 +0x13a fp=0xc421173e48 sp=0xc421173e18

runtime.goparkunlock(0xc42000eef8, 0x4a6d636, 0xc, 0x4b29b17, 0x3)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:265 +0x5e fp=0xc421173e88 sp=0xc421173e48

runtime.chanrecv(0x48ecea0, 0xc42000eea0, 0xc421173f90, 0x1, 0x0)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/chan.go:496 +0x2df fp=0xc421173f10 sp=0xc421173e88

runtime.chanrecv2(0x48ecea0, 0xc42000eea0, 0xc421173f90, 0x0)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/chan.go:383 +0x35 fp=0xc421173f48 sp=0xc421173f10

github.com/couchbase/gometa/protocol.(*Leader).listen(0xc420140e10)

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gometa/protocol/leader.go:472 +0xcd fp=0xc421173fb8 sp=0xc421173f48

runtime.goexit()

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc421173fc0 sp=0xc421173fb8

created by github.com/couchbase/gometa/protocol.NewLeaderWithCustomHandler

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gometa/protocol/leader.go:160 +0x353

 

goroutine 66 [select]:

runtime.gopark(0x4b29a10, 0x0, 0x4a67d1a, 0x6, 0x18, 0x2)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:259 +0x13a fp=0xc4201b2400 sp=0xc4201b23d0

runtime.selectgoImpl(0xc4201b2728, 0x0, 0x18)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/select.go:423 +0x11d9 fp=0xc4201b2628 sp=0xc4201b2400

runtime.selectgo(0xc4201b2728)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/select.go:238 +0x1c fp=0xc4201b2650 sp=0xc4201b2628

github.com/couchbase/indexing/secondary/manager.(*LifecycleMgr).processRequest(0xc42001e580)

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/manager/lifecycle.go:199 +0x3a9 fp=0xc4201b27b8 sp=0xc4201b2650

runtime.goexit()

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc4201b27c0 sp=0xc4201b27b8

created by github.com/couchbase/indexing/secondary/manager.(*LifecycleMgr).Run

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/manager/lifecycle.go:117 +0x73

 

goroutine 32 [IO wait]:

runtime.gopark(0x4b29988, 0x5274dc0, 0x4a68316, 0x7, 0x4258c1b, 0x5)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:259 +0x13a fp=0xc4200448d0 sp=0xc4200448a0

runtime.netpollblock(0x5274d98, 0x72, 0x23)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/netpoll.go:341 +0xeb fp=0xc420044918 sp=0xc4200448d0

net.runtime_pollWait(0x5274d98, 0x72, 0x8)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/netpoll.go:160 +0x59 fp=0xc420044940 sp=0xc420044918

net.(*pollDesc).wait(0xc4201468b0, 0x72, 0xc4200449d0, 0xc420014160)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/net/fd_poll_runtime.go:73 +0x38 fp=0xc420044968 sp=0xc420044940

net.(*pollDesc).waitRead(0xc4201468b0, 0x5044100, 0xc420014160)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/net/fd_poll_runtime.go:78 +0x34 fp=0xc420044998 sp=0xc420044968

net.(*netFD).Read(0xc420146850, 0xc42305e000, 0x1000, 0x1000, 0x0, 0x5044100, 0xc420014160)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/net/fd_unix.go:243 +0x1a1 fp=0xc420044a08 sp=0xc420044998

net.(*conn).Read(0xc42012e0b8, 0xc42305e000, 0x1000, 0x1000, 0x0, 0x0, 0x0)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/net/net.go:173 +0x70 fp=0xc420044a78 sp=0xc420044a08

net/http.(*persistConn).Read(0xc420132500, 0xc42305e000, 0x1000, 0x1000, 0x42bc050, 0xc420044b58, 0x4008a6d)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/net/http/transport.go:1261 +0x154 fp=0xc420044af0 sp=0xc420044a78

bufio.(*Reader).fill(0xc420199740)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/bufio/bufio.go:97 +0x10c fp=0xc420044b50 sp=0xc420044af0

bufio.(*Reader).Peek(0xc420199740, 0x1, 0xc420044bbd, 0x1, 0x0, 0xc4211917a0, 0x0)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/bufio/bufio.go:129 +0x62 fp=0xc420044b68 sp=0xc420044b50

net/http.(*persistConn).readLoop(0xc420132500)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/net/http/transport.go:1418 +0x1a1 fp=0xc420044fa8 sp=0xc420044b68

runtime.goexit()

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc420044fb0 sp=0xc420044fa8

created by net/http.(*Transport).dialConn

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/net/http/transport.go:1062 +0x4e9

 

goroutine 33 [select]:

runtime.gopark(0x4b29a10, 0x0, 0x4a67d1a, 0x6, 0x18, 0x2)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:259 +0x13a fp=0xc423045c08 sp=0xc423045bd8

runtime.selectgoImpl(0xc423045f18, 0x0, 0x18)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/select.go:423 +0x11d9 fp=0xc423045e30 sp=0xc423045c08

runtime.selectgo(0xc423045f18)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/select.go:238 +0x1c fp=0xc423045e58 sp=0xc423045e30

net/http.(*persistConn).writeLoop(0xc420132500)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/net/http/transport.go:1646 +0x3bd fp=0xc423045fa8 sp=0xc423045e58

runtime.goexit()

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc423045fb0 sp=0xc423045fa8

created by net/http.(*Transport).dialConn

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/net/http/transport.go:1063 +0x50e

 

goroutine 55 [IO wait]:

runtime.gopark(0x4b29988, 0x5274d00, 0x4a68316, 0x7, 0xc42007731b, 0x5)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:259 +0x13a fp=0xc421173558 sp=0xc421173528

runtime.netpollblock(0x5274cd8, 0x72, 0x0)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/netpoll.go:341 +0xeb fp=0xc4211735a0 sp=0xc421173558

net.runtime_pollWait(0x5274cd8, 0x72, 0x0)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/netpoll.go:160 +0x59 fp=0xc4211735c8 sp=0xc4211735a0

net.(*pollDesc).wait(0xc420077410, 0x72, 0xc421173678, 0xc420014160)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/net/fd_poll_runtime.go:73 +0x38 fp=0xc4211735f0 sp=0xc4211735c8

net.(*pollDesc).waitRead(0xc420077410, 0x5044100, 0xc420014160)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/net/fd_poll_runtime.go:78 +0x34 fp=0xc421173620 sp=0xc4211735f0

net.(*netFD).accept(0xc4200773b0, 0x0, 0x5042780, 0xc42309f080)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/net/fd_unix.go:419 +0x238 fp=0xc4211736d8 sp=0xc421173620

net.(*TCPListener).accept(0xc4201201c0, 0xc421173760, 0xc421173770, 0x4058600)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/net/tcpsock_posix.go:132 +0x2e fp=0xc421173708 sp=0xc4211736d8

net.(*TCPListener).Accept(0xc4201201c0, 0x4b24fb8, 0xc42013b9c0, 0x0, 0x0)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/net/tcpsock.go:222 +0x49 fp=0xc421173760 sp=0xc421173708

github.com/couchbase/gometa/common.(*PeerListener).listen(0xc42013b9c0)

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gometa/common/peerListener.go:135 +0x62 fp=0xc4211737b8 sp=0xc421173760

runtime.goexit()

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc4211737c0 sp=0xc4211737b8

created by github.com/couchbase/gometa/common.StartPeerListener

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gometa/common/peerListener.go:63 +0x137

 

goroutine 56 [select]:

runtime.gopark(0x4b29a10, 0x0, 0x4a67d1a, 0x6, 0x18, 0x2)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:259 +0x13a fp=0xc420154990 sp=0xc420154960

runtime.selectgoImpl(0xc420154ca0, 0x0, 0x18)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/select.go:423 +0x11d9 fp=0xc420154bb8 sp=0xc420154990

runtime.selectgo(0xc420154ca0)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/select.go:238 +0x1c fp=0xc420154be0 sp=0xc420154bb8

github.com/couchbase/gometa/protocol.(*LeaderServer).processRequest(0xc4220deb80, 0xc420077340, 0xc4230953b0, 0x5046280, 0xc42001e580, 0x0, 0x0)

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gometa/protocol/leaderServer.go:384 +0x657 fp=0xc420154dd0 sp=0xc420154be0

github.com/couchbase/gometa/protocol.RunLeaderServerWithCustomHandler(0xc423086eb0, 0x5, 0xc42013b9c0, 0x5049140, 0xc4220d7e30, 0x50531c0, 0xc4200772d0, 0x5052a20, 0x50aa828, 0x5046280, ...)

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gometa/protocol/leaderServer.go:130 +0x5c9 fp=0xc420154eb8 sp=0xc420154dd0

github.com/couchbase/gometa/server.(*EmbeddedServer).runOnce(0xc4201da640)

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gometa/server/embeddedServer.go:483 +0x166 fp=0xc420154f50 sp=0xc420154eb8

github.com/couchbase/gometa/server.(*EmbeddedServer).run(0xc4201da640)

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gometa/server/embeddedServer.go:385 +0x2f fp=0xc420154fa8 sp=0xc420154f50

runtime.goexit()

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc420154fb0 sp=0xc420154fa8

created by github.com/couchbase/gometa/server.RunEmbeddedServerWithCustomHandler

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gometa/server/embeddedServer.go:82 +0x206

 

goroutine 67 [select]:

runtime.gopark(0x4b29a10, 0x0, 0x4a67d1a, 0x6, 0x18, 0x2)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:259 +0x13a fp=0xc420041c20 sp=0xc420041bf0

runtime.selectgoImpl(0xc420041f10, 0x0, 0x18)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/select.go:423 +0x11d9 fp=0xc420041e48 sp=0xc420041c20

runtime.selectgo(0xc420041f10)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/select.go:238 +0x1c fp=0xc420041e70 sp=0xc420041e48

github.com/couchbase/indexing/secondary/manager.(*IndexManager).monitorBucket(0xc420146690, 0xc4211907e0)

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/manager/manager.go:568 +0x2c5 fp=0xc420041fa0 sp=0xc420041e70

runtime.goexit()

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc420041fa8 sp=0xc420041fa0

created by github.com/couchbase/indexing/secondary/manager.NewIndexManagerInternal

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/manager/manager.go:186 +0x786

 

goroutine 68 [chan receive]:

runtime.gopark(0x4b299d0, 0xc4211904d8, 0x4a6d636, 0xc, 0x17, 0x3)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:259 +0x13a fp=0xc42339be40 sp=0xc42339be10

runtime.goparkunlock(0xc4211904d8, 0x4a6d636, 0xc, 0x17, 0x3)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:265 +0x5e fp=0xc42339be80 sp=0xc42339be40

runtime.chanrecv(0x4926820, 0xc421190480, 0xc42339bf88, 0xc4201b3701, 0xc4201b0101)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/chan.go:496 +0x2df fp=0xc42339bf08 sp=0xc42339be80

runtime.chanrecv2(0x4926820, 0xc421190480, 0xc42339bf88, 0x1)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/chan.go:383 +0x35 fp=0xc42339bf40 sp=0xc42339bf08

github.com/couchbase/indexing/secondary/indexer.(*clustMgrAgent).run(0xc420126ec0)

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/cluster_manager_agent.go:99 +0xbb fp=0xc42339bfa8 sp=0xc42339bf40

runtime.goexit()

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc42339bfb0 sp=0xc42339bfa8

created by github.com/couchbase/indexing/secondary/indexer.NewClustMgrAgent

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/cluster_manager_agent.go:77 +0x18e

 

goroutine 69 [chan receive]:

runtime.gopark(0x4b299d0, 0xc4211905f8, 0x4a6d636, 0xc, 0x17, 0x3)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:259 +0x13a fp=0xc4201b3e50 sp=0xc4201b3e20

runtime.goparkunlock(0xc4211905f8, 0x4a6d636, 0xc, 0x4056a17, 0x3)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:265 +0x5e fp=0xc4201b3e90 sp=0xc4201b3e50

runtime.chanrecv(0x4926820, 0xc4211905a0, 0xc4201b3f98, 0x503e501, 0x50aa828)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/chan.go:496 +0x2df fp=0xc4201b3f18 sp=0xc4201b3e90

runtime.chanrecv2(0x4926820, 0xc4211905a0, 0xc4201b3f98, 0x1)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/chan.go:383 +0x35 fp=0xc4201b3f50 sp=0xc4201b3f18

github.com/couchbase/indexing/secondary/indexer.(*statsManager).run(0xc42001cd70)

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/stats_manager.go:633 +0x64 fp=0xc4201b3fb8 sp=0xc4201b3f50

runtime.goexit()

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc4201b3fc0 sp=0xc4201b3fb8

created by github.com/couchbase/indexing/secondary/indexer.NewStatsManager

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/stats_manager.go:471 +0x3c3

 

goroutine 70 [sleep]:

runtime.gopark(0x4b299d0, 0x50b42e0, 0x4a66aa0, 0x5, 0x13, 0x2)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:259 +0x13a fp=0xc4201b55f0 sp=0xc4201b55c0

runtime.goparkunlock(0x50b42e0, 0x4a66aa0, 0x5, 0xc420120113, 0x2)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:265 +0x5e fp=0xc4201b5630 sp=0xc4201b55f0

time.Sleep(0xdf8475800)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/time.go:59 +0xe1 fp=0xc4201b5670 sp=0xc4201b5630

github.com/couchbase/indexing/secondary/indexer.(*statsManager).runStatsDumpLogger(0xc42001cd70)

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/stats_manager.go:679 +0x9e fp=0xc4201b57b8 sp=0xc4201b5670

runtime.goexit()

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc4201b57c0 sp=0xc4201b57b8

created by github.com/couchbase/indexing/secondary/indexer.NewStatsManager

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/stats_manager.go:472 +0x3e5

 

goroutine 71 [chan receive]:

runtime.gopark(0x4b299d0, 0xc42000f018, 0x4a6d636, 0xc, 0x17, 0x3)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:259 +0x13a fp=0xc4201b4608 sp=0xc4201b45d8

runtime.goparkunlock(0xc42000f018, 0x4a6d636, 0xc, 0x17, 0x3)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:265 +0x5e fp=0xc4201b4648 sp=0xc4201b4608

runtime.chanrecv(0x48ecca0, 0xc42000efc0, 0xc4201b4770, 0xc4201b4701, 0xc4201b4718)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/chan.go:496 +0x2df fp=0xc4201b46d0 sp=0xc4201b4648

runtime.chanrecv2(0x48ecca0, 0xc42000efc0, 0xc4201b4770, 0xc42013a100)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/chan.go:383 +0x35 fp=0xc4201b4708 sp=0xc4201b46d0

github.com/couchbase/indexing/secondary/indexer.(*cpuCollector).runCollectCpu(0xc420032030)

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/cpu.go:64 +0xd0 fp=0xc4201b47b8 sp=0xc4201b4708

runtime.goexit()

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc4201b47c0 sp=0xc4201b47b8

created by github.com/couchbase/indexing/secondary/indexer.StartCpuCollector

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/cpu.go:48 +0xa4

 

goroutine 72 [chan receive]:

runtime.gopark(0x4b299d0, 0xc421190898, 0x4a6d636, 0xc, 0x17, 0x3)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:259 +0x13a fp=0xc4201b5e20 sp=0xc4201b5df0

runtime.goparkunlock(0xc421190898, 0x4a6d636, 0xc, 0x17, 0x3)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:265 +0x5e fp=0xc4201b5e60 sp=0xc4201b5e20

runtime.chanrecv(0x48ecca0, 0xc421190840, 0x0, 0x1, 0x0)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/chan.go:496 +0x2df fp=0xc4201b5ee8 sp=0xc4201b5e60

runtime.chanrecv1(0x48ecca0, 0xc421190840, 0x0)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/chan.go:378 +0x35 fp=0xc4201b5f20 sp=0xc4201b5ee8

github.com/couchbase/indexing/secondary/logging.PeriodicProfile.func1(0xc42005fa00, 0xc420150007, 0xc4230b46d8, 0x5, 0xc4230b46e0, 0x1, 0x1)

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/logging/logging.go:196 +0x58 fp=0xc4201b5f88 sp=0xc4201b5f20

runtime.goexit()

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc4201b5f90 sp=0xc4201b5f88

created by github.com/couchbase/indexing/secondary/logging.PeriodicProfile

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/logging/logging.go:200 +0x95

 

goroutine 73 [IO wait]:

runtime.gopark(0x4b29988, 0x5274c40, 0x4a68316, 0x7, 0xc42007751b, 0x5)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:259 +0x13a fp=0xc420043bc8 sp=0xc420043b98

runtime.netpollblock(0x5274c18, 0x72, 0x0)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/netpoll.go:341 +0xeb fp=0xc420043c10 sp=0xc420043bc8

net.runtime_pollWait(0x5274c18, 0x72, 0x0)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/netpoll.go:160 +0x59 fp=0xc420043c38 sp=0xc420043c10

net.(*pollDesc).wait(0xc420077560, 0x72, 0xc420043ce8, 0xc420014160)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/net/fd_poll_runtime.go:73 +0x38 fp=0xc420043c60 sp=0xc420043c38

net.(*pollDesc).waitRead(0xc420077560, 0x5044100, 0xc420014160)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/net/fd_poll_runtime.go:78 +0x34 fp=0xc420043c90 sp=0xc420043c60

net.(*netFD).accept(0xc420077500, 0x0, 0x5042780, 0xc4230b0380)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/net/fd_unix.go:419 +0x238 fp=0xc420043d48 sp=0xc420043c90

net.(*TCPListener).accept(0xc4201201d0, 0xc4230a6900, 0xc420043df8, 0x46c35d0)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/net/tcpsock_posix.go:132 +0x2e fp=0xc420043d78 sp=0xc420043d48

net.(*TCPListener).AcceptTCP(0xc4201201d0, 0xc42001eadc, 0xc420043de8, 0x42bb7f0)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/net/tcpsock.go:209 +0x49 fp=0xc420043dd0 sp=0xc420043d78

net/http.tcpKeepAliveListener.Accept(0xc4201201d0, 0xc4230a68d0, 0x492d520, 0x5094b80, 0x498e640)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/net/http/server.go:2608 +0x2f fp=0xc420043e08 sp=0xc420043dd0

net/http.(*Server).Serve(0xc42001ea80, 0x504aa80, 0xc4201201d0, 0x0, 0x0)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/net/http/server.go:2273 +0x1ce fp=0xc420043ec0 sp=0xc420043e08

net/http.(*Server).ListenAndServe(0xc42001ea80, 0xc42001ea80, 0x0)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/net/http/server.go:2219 +0xb4 fp=0xc420043f10 sp=0xc420043ec0

net/http.ListenAndServe(0xc4230b46d8, 0x5, 0x0, 0x0, 0x0, 0x0)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/net/http/server.go:2351 +0xa0 fp=0xc420043f40 sp=0xc420043f10

github.com/couchbase/indexing/secondary/indexer.NewIndexer.func1(0xc4230b46d8, 0x5)

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:296 +0x4b fp=0xc420043fa0 sp=0xc420043f40

runtime.goexit()

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc420043fa8 sp=0xc420043fa0

created by github.com/couchbase/indexing/secondary/indexer.NewIndexer

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:300 +0x25f0

 

goroutine 74 [IO wait]:

runtime.gopark(0x4b29988, 0x5274b80, 0x4a68316, 0x7, 0xc42007761b, 0x5)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:259 +0x13a fp=0xc420051998 sp=0xc420051968

runtime.netpollblock(0x5274b58, 0x72, 0x0)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/netpoll.go:341 +0xeb fp=0xc4200519e0 sp=0xc420051998

net.runtime_pollWait(0x5274b58, 0x72, 0x0)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/netpoll.go:160 +0x59 fp=0xc420051a08 sp=0xc4200519e0

net.(*pollDesc).wait(0xc420077720, 0x72, 0xc42014daa8, 0xc420014160)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/net/fd_poll_runtime.go:73 +0x38 fp=0xc420051a30 sp=0xc420051a08

net.(*pollDesc).waitRead(0xc420077720, 0x5044100, 0xc420014160)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/net/fd_poll_runtime.go:78 +0x34 fp=0xc420051a60 sp=0xc420051a30

net.(*netFD).accept(0xc4200776c0, 0x0, 0x5042780, 0xc4230e57e0)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/net/fd_unix.go:419 +0x238 fp=0xc420051b18 sp=0xc420051a60

net.(*TCPListener).accept(0xc4201201e8, 0x4014ec8, 0x30, 0x49bf0e0)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/net/tcpsock_posix.go:132 +0x2e fp=0xc420051b48 sp=0xc420051b18

net.(*TCPListener).Accept(0xc4201201e8, 0xc400000001, 0xc4230a6e40, 0x492d520, 0x5094b80)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/net/tcpsock.go:222 +0x49 fp=0xc420051ba0 sp=0xc420051b48

crypto/tls.(*listener).Accept(0xc4230e5760, 0xc4230a6e10, 0x492d520, 0x5094b80, 0x498e640)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/crypto/tls/tls.go:52 +0x37 fp=0xc420051bf8 sp=0xc420051ba0

net/http.(*Server).Serve(0xc42001ed80, 0x5048d40, 0xc4230e5760, 0x0, 0x0)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/net/http/server.go:2273 +0x1ce fp=0xc420051cb0 sp=0xc420051bf8

net/http.Serve(0x5048d40, 0xc4230e5760, 0x0, 0x0, 0x1, 0xc4201201e8)

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/net/http/server.go:2091 +0x99 fp=0xc420051cf0 sp=0xc420051cb0

github.com/couchbase/indexing/secondary/indexer.NewIndexer.func3(0x7fff5fbff45f, 0x6c, 0x7fff5fbff4d6, 0x6c, 0xc4230b46f0, 0x6, 0xc420032040, 0xc4230b46f6)

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:348 +0x4bf fp=0xc420051f60 sp=0xc420051cf0

runtime.goexit()

/Users/jenkins/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc420051f68 sp=0xc420051f60

created by github.com/couchbase/indexing/secondary/indexer.NewIndexer

/Users/jenkins/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:356 +0x39bf