[BP 5.5.3] RollbackToZero in slice writer needs to use waitPersist()

2018-10-01T21:05:39.795+00:00 [Info] StorageMgr::handleRollback Rollback Index: 5566340690216382524 PartitionId: 0 SliceId: 0 To Zero 

panic: FDB_RESULT_FAIL_BY_ROLLBACK

goroutine 6589 [running]:

panic(0xcc5280, 0xc47d844038)

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/panic.go:500 +0x1a1 fp=0xc4254b89e8 sp=0xc4254b8958

github.com/couchbase/indexing/secondary/common.CrashOnError(0x15a1da0, 0xc47d844038)

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/common/util.go:402 +0x5c fp=0xc4254b8a18 sp=0xc4254b89e8

github.com/couchbase/indexing/secondary/indexer.(*fdbSlice).checkFatalDbError(0xc42326e000, 0x15a1da0, 0xc47d844038)

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/forestdb_slice_writer.go:887 +0x39 fp=0xc4254b8a60 sp=0xc4254b8a18

github.com/couchbase/indexing/secondary/indexer.(*fdbSlice).insertSecIndex(0xc42326e000, 0xc4554a3680, 0x83, 0x90, 0xc42e822b90, 0x45, 0x45, 0x0, 0x1a950a17)

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/forestdb_slice_writer.go:472 +0x6df fp=0xc4254b8d48 sp=0xc4254b8a60

github.com/couchbase/indexing/secondary/indexer.(*fdbSlice).insert(0xc42326e000, 0xc4554a3680, 0x83, 0x90, 0xc45c721880, 0x3c, 0x40, 0xc42e822b90, 0x45, 0x45, ...)

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/forestdb_slice_writer.go:360 +0x162 fp=0xc4254b8dc0 sp=0xc4254b8d48

github.com/couchbase/indexing/secondary/indexer.(*fdbSlice).handleCommandsWorker(0xc42326e000, 0x0)

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/forestdb_slice_writer.go:320 +0x24f fp=0xc4254b8fa0 sp=0xc4254b8dc0

runtime.goexit()

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc4254b8fa8 sp=0xc4254b8fa0

created by github.com/couchbase/indexing/secondary/indexer.NewForestDBSlice

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/forestdb_slice_writer.go:160 +0x1243

goroutine 1 [chan receive]:

runtime.gopark(0xeb0ab8, 0xc4211d82f8, 0xdf91ed, 0xc, 0x40b017, 0x3)

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:259 +0x13a fp=0xc426fe4858 sp=0xc426fe4828

runtime.goparkunlock(0xc4211d82f8, 0xdf91ed, 0xc, 0x17, 0x3)

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:265 +0x5e fp=0xc426fe4898 sp=0xc426fe4858

runtime.chanrecv(0xcc5340, 0xc4211d82a0, 0xc426fe49c8, 0x507a01, 0xc48bce0101)

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/chan.go:496 +0x2df fp=0xc426fe4920 sp=0xc426fe4898

runtime.chanrecv1(0xcc5340, 0xc4211d82a0, 0xc426fe49c8)

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/chan.go:378 +0x35 fp=0xc426fe4958 sp=0xc426fe4920

github.com/couchbase/indexing/secondary/indexer.(*indexer).processRollback(0xc42014e000, 0xc42d5f0001, 0xc423cf3a70, 0xa, 0xc442591f10, 0x1, 0xc4245f64f8, 0x2)

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:3303 +0x194 fp=0xc426fe4a68 sp=0xc426fe4958

github.com/couchbase/indexing/secondary/indexer.(*indexer).handleInitRecovery(0xc42014e000, 0x159cea0, 0xc4cede6c80)

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:1701 +0x2f1 fp=0xc426fe4b20 sp=0xc426fe4a68

github.com/couchbase/indexing/secondary/indexer.(*indexer).handleWorkerMsgs(0xc42014e000, 0x159cea0, 0xc4cede6c80)

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:813 +0x1914 fp=0xc426fe4e48 sp=0xc426fe4b20

github.com/couchbase/indexing/secondary/indexer.(*indexer).run(0xc42014e000)

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:593 +0x17b fp=0xc426fe4f78 sp=0xc426fe4e48

github.com/couchbase/indexing/secondary/indexer.NewIndexer(0xc4200186c0, 0x24, 0xc420057df8, 0x1, 0x1)

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:450 +0x3221 fp=0xc426fe5c80 sp=0xc426fe4f78

main.main()

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/cmd/indexer/main.go:161 +0x1221 fp=0xc426fe5f38 sp=0xc426fe5c80

runtime.main()

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/proc.go:183 +0x1f4 fp=0xc426fe5f90 sp=0xc426fe5f38

runtime.goexit()

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.3/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc426fe5f98 sp=0xc426fe5f90