CBAuth database is stale

Test failed:

./testrunner -i /tmp/centos6_upgrade.ini get-cbcollect-info=False,get-logs=False,get-coredumps=False -t 2i.offset_enhancement_2i.SecondaryIndexingOffsetTests.test_with_offset_0_and_limit_0,doc-per-day=100,dataset=array,nodes_init=2,services_init=kv:n1ql-kv:index

error seen in projector log:
017-01-16T00:08:42.871-08:00 [Error] GetSettingsConfig() failed: Get http://127.0.0.1:8091/_metakv/indexing/settings/config: CBAuth database is stale: last reason: Need 200 status!. Got {401 Unauthorized 401 HTTP/1.1 1 1 map[Www-Authenticate:[Basic realm="Couchbase Server Admin / REST"] Server:[Couchbase Server] Pragma:[no-cache] Date:[Mon, 16 Jan 2017 08:08:42 GMT] Content-Length:[0] Cache-Control:[no-cache]] 0x16b0d30 0 [] false map[] 0xc820180000 <nil>}

panic: Get http://127.0.0.1:8091/_metakv/indexing/settings/config: CBAuth database is stale: last reason: Need 200 status!. Got {401 Unauthorized 401 HTTP/1.1 1 1 map[Www-Authenticate:[Basic realm="Couchbase Server Admin / REST"] Server:[Couchbase Server] Pragma:[no-cache] Date:[Mon, 16 Jan 2017 08:08:42 GMT] Content-Length:[0] Cache-Control:[no-cache]] 0x16b0d30 0 [] false map[] 0xc820180000 <nil>}

 

goroutine 1 [running]:

panic(0xd9b580, 0xc82013b4a0)

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/panic.go:481 +0x3e6 fp=0xc820139a20 sp=0xc8201399a0

github.com/couchbase/indexing/secondary/common.CrashOnError(0x7fec5971bcb8, 0xc82013b4a0)

/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/util.go:368 +0x4b fp=0xc820139a48 sp=0xc820139a20

github.com/couchbase/indexing/secondary/projector.NewProjector(0x400, 0xc82013b3b0, 0xc820139e88)

/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/projector/projector.go:54 +0x17e fp=0xc820139d98 sp=0xc820139a48

main.main()

/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/cmd/projector/main.go:106 +0xaef fp=0xc820139f40 sp=0xc820139d98

runtime.main()

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/proc.go:188 +0x2b0 fp=0xc820139f90 sp=0xc820139f40

runtime.goexit()

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/asm_amd64.s:1998 +0x1 fp=0xc820139f98 sp=0xc820139f90

 

goroutine 17 [syscall, locked to thread]:

runtime.goexit()

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/asm_amd64.s:1998 +0x1 fp=0xc82003efb8 sp=0xc82003efb0

 

goroutine 2 [force gc (idle)]:

runtime.gopark(0x10aaff0, 0x16b1360, 0xec6660, 0xf, 0x14, 0x1)

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/proc.go:262 +0x163 fp=0xc82002a758 sp=0xc82002a730

runtime.goparkunlock(0x16b1360, 0xec6660, 0xf, 0xc820000114, 0x1)

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/proc.go:268 +0x54 fp=0xc82002a790 sp=0xc82002a758

runtime.forcegchelper()

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/proc.go:229 +0xb8 fp=0xc82002a7c0 sp=0xc82002a790

runtime.goexit()

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/asm_amd64.s:1998 +0x1 fp=0xc82002a7c8 sp=0xc82002a7c0

created by runtime.init.4

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/proc.go:218 +0x2b

 

goroutine 18 [GC sweep wait]:

runtime.gopark(0x10aaff0, 0x16b15c0, 0xebae80, 0xd, 0x422c14, 0x1)

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/proc.go:262 +0x163 fp=0xc820026748 sp=0xc820026720

runtime.goparkunlock(0x16b15c0, 0xebae80, 0xd, 0x14, 0x1)

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/proc.go:268 +0x54 fp=0xc820026780 sp=0xc820026748

runtime.bgsweep(0xc82006e000)

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/mgcsweep.go:63 +0xb1 fp=0xc8200267b8 sp=0xc820026780

runtime.goexit()

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/asm_amd64.s:1998 +0x1 fp=0xc8200267c0 sp=0xc8200267b8

created by runtime.gcenable

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/mgc.go:191 +0x53

 

goroutine 19 [finalizer wait]:

runtime.gopark(0x10aaff0, 0x16d0640, 0xec61b0, 0xe, 0x14, 0x1)

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/proc.go:262 +0x163 fp=0xc820026f18 sp=0xc820026ef0

runtime.goparkunlock(0x16d0640, 0xec61b0, 0xe, 0x14, 0x1)

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/proc.go:268 +0x54 fp=0xc820026f50 sp=0xc820026f18

runtime.runfinq()

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/mfinal.go:158 +0xaa fp=0xc820026fc0 sp=0xc820026f50

runtime.goexit()

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/asm_amd64.s:1998 +0x1 fp=0xc820026fc8 sp=0xc820026fc0

created by runtime.createfing

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/mfinal.go:139 +0x60

 

goroutine 21 [syscall]:

runtime.notetsleepg(0x16b1878, 0x3b8bcef6, 0x409501)

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/lock_futex.go:205 +0x4e fp=0xc820027738 sp=0xc820027710

runtime.timerproc()

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/time.go:209 +0xde fp=0xc8200277c0 sp=0xc820027738

runtime.goexit()

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/asm_amd64.s:1998 +0x1 fp=0xc8200277c8 sp=0xc8200277c0

created by runtime.addtimerLocked

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/time.go:116 +0x11f

 

goroutine 22 [sleep]:

runtime.gopark(0x10aaff0, 0x16b1860, 0xeb4240, 0x5, 0x13, 0x2)

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/proc.go:262 +0x163 fp=0xc820135cb0 sp=0xc820135c88

runtime.goparkunlock(0x16b1860, 0xeb4240, 0x5, 0x113, 0x2)

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/proc.go:268 +0x54 fp=0xc820135ce8 sp=0xc820135cb0

time.Sleep(0x3b9aca00)

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/time.go:59 +0xf9 fp=0xc820135d30 sp=0xc820135ce8

github.com/couchbase/cbauth/revrpc.(*DefaultErrorPolicy).try(0xc820142000, 0x7fec59709000, 0xc820190110, 0x0, 0x0)

/home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:205 +0x520 fp=0xc820135e00 sp=0xc820135d30

github.com/couchbase/cbauth/revrpc.(*DefaultErrorPolicy).(github.com/couchbase/cbauth/revrpc.try)-fm(0x7fec59709000, 0xc820190110, 0x0, 0x0)

/home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:217 +0x42 fp=0xc820135e30 sp=0xc820135e00

github.com/couchbase/cbauth.runRPCForSvc.func1(0x7fec59709000, 0xc820190110, 0x0, 0x0)

/home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/default.go:54 +0xf6 fp=0xc820135e90 sp=0xc820135e30

github.com/couchbase/cbauth/revrpc.BabysitService(0xc82014e010, 0xc82013a270, 0x7fec5970d858, 0xc820142020, 0x0, 0x0)

/home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:242 +0x9e fp=0xc820135ee0 sp=0xc820135e90

github.com/couchbase/cbauth.runRPCForSvc(0xc82013a270, 0xc8200716c0, 0x0, 0x0)

/home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/default.go:58 +0x11b fp=0xc820135f48 sp=0xc820135ee0

github.com/couchbase/cbauth.startDefault.func1(0xc82013a270, 0xc8200716c0)

/home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/default.go:65 +0x2b fp=0xc820135f80 sp=0xc820135f48

runtime.goexit()

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/asm_amd64.s:1998 +0x1 fp=0xc820135f88 sp=0xc820135f80

created by github.com/couchbase/cbauth.startDefault

/home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/default.go:66 +0x120

 

goroutine 23 [sleep]:

runtime.gopark(0x10aaff0, 0x16b1860, 0xeb4240, 0x5, 0x13, 0x2)

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/proc.go:262 +0x163 fp=0xc8200286b8 sp=0xc820028690

runtime.goparkunlock(0x16b1860, 0xeb4240, 0x5, 0x13, 0x2)

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/proc.go:268 +0x54 fp=0xc8200286f0 sp=0xc8200286b8

time.Sleep(0x2540be400)

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/time.go:59 +0xf9 fp=0xc820028738 sp=0xc8200286f0

github.com/couchbase/indexing/secondary/common.pollForDeletedBuckets()

/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/dcp_seqno.go:341 +0x3a fp=0xc8200287c0 sp=0xc820028738

runtime.goexit()

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/asm_amd64.s:1998 +0x1 fp=0xc8200287c8 sp=0xc8200287c0

created by github.com/couchbase/indexing/secondary/common.init.1

/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/dcp_seqno.go:34 +0xf3

 

goroutine 25 [syscall]:

runtime.notetsleepg(0x16d0a80, 0xffffffffffffffff, 0x1)

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/lock_futex.go:205 +0x4e fp=0xc820029740 sp=0xc820029718

os/signal.signal_recv(0x0)

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/sigqueue.go:116 +0x132 fp=0xc820029778 sp=0xc820029740

os/signal.loop()

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/os/signal/signal_unix.go:22 +0x18 fp=0xc8200297c0 sp=0xc820029778

runtime.goexit()

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/asm_amd64.s:1998 +0x1 fp=0xc8200297c8 sp=0xc8200297c0

created by os/signal.init.1

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/os/signal/signal_unix.go:28 +0x37

 

goroutine 26 [syscall]:

syscall.Syscall(0x0, 0x0, 0xc820029f44, 0x4, 0x0, 0x0, 0x0)

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/syscall/asm_linux_amd64.s:18 +0x5 fp=0xc820029df0 sp=0xc820029de8

syscall.read(0x0, 0xc820029f44, 0x4, 0x4, 0x0, 0x0, 0x0)

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/syscall/zsyscall_linux_amd64.go:783 +0x5f fp=0xc820029e30 sp=0xc820029df0

syscall.Read(0x0, 0xc820029f44, 0x4, 0x4, 0x0, 0x0, 0x0)

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/syscall/syscall_unix.go:161 +0x4d fp=0xc820029e70 sp=0xc820029e30

os.(*File).read(0xc820084000, 0xc820029f44, 0x4, 0x4, 0x0, 0x0, 0x0)

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/os/file_unix.go:228 +0x53 fp=0xc820029eb0 sp=0xc820029e70

os.(*File).Read(0xc820084000, 0xc820029f44, 0x4, 0x4, 0x0, 0x0, 0x0)

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/os/file.go:95 +0x8a fp=0xc820029f08 sp=0xc820029eb0

github.com/couchbase/indexing/secondary/common.ExitOnStdinClose()

/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/util.go:319 +0x97 fp=0xc820029fc0 sp=0xc820029f08

runtime.goexit()

/home/couchbase/.cbdepscache/exploded/x86_64/go-1.6.3/go/src/runtime/asm_amd64.s:1998 +0x1 fp=0xc820029fc8 sp=0xc820029fc0

created by main.main

/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/cmd/projector/main.go:105 +0xad5

[goport] 2017/01/16 00:08:42 /opt/couchbase/bin/projector terminated: signal: aborted

2017-01-16T00:08:48.164-08:00 [Info] Projector started with command line: [/opt/couchbase/bin/projector -kvaddrs=127.0.0.1:11210 -adminport=:9999 -diagDir=/opt/couchbase/var/lib/couchbase/crash 127.0.0.1:8091]

2017-01-16T00:08:48.165-08:00 [Info] Parsing the args

 

Will upload the logs.