Details
-
Bug
-
Resolution: Unresolved
-
Critical
-
5.0.0
-
None
-
Enterprise Edition 5.0.0 build 3456
-
Untriaged
-
No
Description
I was trying out the steps mentioned below from MB-22040.
a. reduce max file-descriptors limit to say 1000
b. enable backfill.
c. start issuing range queries on indexes that will take long time to complete. Keep increasing the concurrent queries until we hit file-descriptor limit.
d. without the fix, you will see panic in query.log, with fix you will see a log message but no panic.
When I ran into the following 2 panics.
panic: Inconsistent Internal State
|
|
|
goroutine 1 [running]:
|
panic(0xd27ac0, 0xc420127550)
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/panic.go:500 +0x1a1 fp=0xc426397ef8 sp=0xc426397e68
|
github.com/couchbase/indexing/secondary/common.CrashOnError(0x1650ec0, 0xc420127550)
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/util.go:402 +0x5c fp=0xc426397f28 sp=0xc426397ef8
|
github.com/couchbase/indexing/secondary/indexer.(*indexer).sendStreamUpdateForBuildIndex(0xc4200fc480, 0xc42b7ea448, 0x1, 0x1, 0x3, 0xc423b19c80, 0xb, 0xc42b384000, 0x400, 0x400, ...)
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:2088 +0x7d2 fp=0xc426398530 sp=0xc426397f28
|
github.com/couchbase/indexing/secondary/indexer.(*indexer).handleBuildIndex(0xc4200fc480, 0x1651600, 0xc42b860f80)
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:1414 +0xf72 fp=0xc426398d98 sp=0xc426398530
|
github.com/couchbase/indexing/secondary/indexer.(*indexer).handleAdminMsgs(0xc4200fc480, 0x1651600, 0xc42b860f80)
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:1025 +0x1bd fp=0xc426398e48 sp=0xc426398d98
|
github.com/couchbase/indexing/secondary/indexer.(*indexer).run(0xc4200fc480)
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:587 +0x1e8 fp=0xc426398f78 sp=0xc426398e48
|
github.com/couchbase/indexing/secondary/indexer.NewIndexer(0xc420104510, 0x24, 0xc42004ddf8, 0x1, 0x1)
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:439 +0x3295 fp=0xc426399c70 sp=0xc426398f78
|
main.main()
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/cmd/indexer/main.go:161 +0x1221 fp=0xc426399f28 sp=0xc426399c70
|
runtime.main()
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:183 +0x1f4 fp=0xc426399f80 sp=0xc426399f28
|
runtime.goexit()
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc426399f88 sp=0xc426399f80
|
|
|
goroutine 17 [syscall, 89 minutes, locked to thread]:
|
runtime.goexit()
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc42003cfb8 sp=0xc42003cfb0
|
panic: accept tcp [::]:9102: accept4: too many open files in system
|
|
|
goroutine 44 [running]:
|
panic(0xd52dc0, 0xc42dfb0af0)
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/panic.go:500 +0x1a1 fp=0xc422fbdf10 sp=0xc422fbde80
|
github.com/couchbase/indexing/secondary/common.CrashOnError(0x16555c0, 0xc42dfb0af0)
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/util.go:402 +0x5c fp=0xc422fbdf40 sp=0xc422fbdf10
|
github.com/couchbase/indexing/secondary/indexer.NewIndexer.func1(0xc4230e6ab8, 0x5)
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:293 +0xef fp=0xc422fbdfa0 sp=0xc422fbdf40
|
runtime.goexit()
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc422fbdfa8 sp=0xc422fbdfa0
|
created by github.com/couchbase/indexing/secondary/indexer.NewIndexer
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:295 +0x251c
|
|
|
goroutine 1 [select]:
|
runtime.gopark(0xf26720, 0x0, 0xe64c37, 0x6, 0x18, 0x2)
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:259 +0x13a fp=0xc432606c08 sp=0xc432606bd8
|
runtime.selectgoImpl(0xc432606ec0, 0x0, 0x18)
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/select.go:423 +0x11d9 fp=0xc432606e30 sp=0xc432606c08
|
runtime.selectgo(0xc432606ec0)
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/select.go:238 +0x1c fp=0xc432606e58 sp=0xc432606e30
|
github.com/couchbase/indexing/secondary/indexer.(*indexer).run(0xc420100480)
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:578 +0x291 fp=0xc432606f88 sp=0xc432606e58
|
github.com/couchbase/indexing/secondary/indexer.NewIndexer(0xc42010c510, 0x24, 0xc42004ddf8, 0x1, 0x1)
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:439 +0x3295 fp=0xc432607c80 sp=0xc432606f88
|
main.main()
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/cmd/indexer/main.go:161 +0x1221 fp=0xc432607f38 sp=0xc432607c80
|
runtime.main()
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:183 +0x1f4 fp=0xc432607f90 sp=0xc432607f38
|
runtime.goexit()
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc432607f98 sp=0xc432607f90
|
|
Seems like both of them are realted. Logs attached.