[FTS] Forestdb assertion fails, cbft crashes repeatedly

Build
4.5.0-1673

This one is definitely a regression, not too sure if this is related to MOSS.

There are 21 indexes, each with 1K keys. Do not see high memory consumption so isolating from the famous OOM killer bug. I see fts constantly crashing on .145. Looking into logs, I find -

2016-03-03T15:41:09.174-08:00 [INFO] ctl/manager, GetCurrenTopology, haveTopologyRev: , changed, rv: &service.Topology{Rev:service.Revision{0x32}, Nodes:[]service.NodeID{"94747db3edc096ca94f228a3059e8cf9", "9d1b09ec95a33fc01e0f8c99fe186ad2", "a175844f5f784e6f2e97b523bfb4b2b6"}, IsBalanced:false, Messages:[]string(nil)}

Assertion in (nil) != 0xc in /home/couchbase/serverjenkins/workspace/watson-unix/forestdb/src/hbtrie.cc:1159

cbft: /home/couchbase/serverjenkins/workspace/watson-unix/forestdb/src/hbtrie.cc:1159: hbtrie_result _hbtrie_find(hbtrie*, void*, int, void*, list*, uint8_t): Assertion `btree->ksize == trie->chunksize && btree->vsize == trie->valuelen' failed.

SIGABRT: abort

PC=0x7f8e17151625 m=8

signal arrived during cgo execution

Full stack-trace:
2016-03-03T15:41:09.032-08:00 [INFO] ------------------------------------------------------------
2016-03-03T15:41:09.032-08:00 [INFO] web UI / REST API is available (via 0.0.0.0): http://172.23.107.145:8094
2016-03-03T15:41:09.032-08:00 [INFO] ------------------------------------------------------------
2016-03-03T15:41:09.032-08:00 [INFO] main: cbauth_service registering...
2016-03-03T15:41:09.032-08:00 [INFO] ------------------------------------------------------------
2016-03-03T15:41:09.032-08:00 [INFO] web UI / REST API is available: http://0.0.0.0:8094
2016-03-03T15:41:09.032-08:00 [INFO] ------------------------------------------------------------
2016-03-03T15:41:09.167-08:00 [INFO] ctl/manager, GetTaskList, haveTasksRev: , changed, rv: &service.TaskList{Rev:service.Revision

, Tasks:[]service.Task{}}
2016-03-03T15:41:09.174-08:00 [INFO] ctl/manager, GetCurrenTopology, haveTopologyRev: , changed, rv: &service.Topology{Rev:service.Revision

, Nodes:[]service.NodeID

, IsBalanced:false, Messages:[]string(nil)}
Assertion in (nil) != 0xc in /home/couchbase/serverjenkins/workspace/watson-unix/forestdb/src/hbtrie.cc:1159
cbft: /home/couchbase/serverjenkins/workspace/watson-unix/forestdb/src/hbtrie.cc:1159: hbtrie_result _hbtrie_find(hbtrie*, void*, int, void*, list*, uint8_t): Assertion `btree->ksize == trie->chunksize && btree->vsize == trie->valuelen' failed.
SIGABRT: abort
PC=0x7f8e17151625 m=8
signal arrived during cgo execution

goroutine 67 [syscall, locked to thread]:
runtime.cgocall(0xd41be0, 0xc8209cb010, 0xc800000000)
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/runtime/cgocall.go:120 +0x11b fp=0xc8209cafc8 sp=0xc8209caf98
github.com/couchbase/goforestdb._Cfunc_fdb_get_kv(0x7f8dd8011ec0, 0x16e4168, 0x1, 0xc8210965a8, 0xc8210c3480, 0x0)
??:0 +0x39 fp=0xc8209cb010 sp=0xc8209cafc8
github.com/couchbase/goforestdb.(*KVStore).GetKV(0xc82072c580, 0x16e4168, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/couchbase/serverjenkins/workspace/watson-unix/godeps/src/github.com/couchbase/goforestdb/kv.go:33 +0x23d fp=0xc8209cb0d8 sp=0xc8209cb010
github.com/blevesearch/blevex/forestdb.(*Reader).Get(0xc82072c5a0, 0x16e4168, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/couchbase/serverjenkins/workspace/watson-unix/godeps/src/github.com/blevesearch/blevex/forestdb/reader.go:24 +0x7f fp=0xc8209cb158 sp=0xc8209cb0d8
github.com/blevesearch/bleve/index/store/moss.(*llSnapshot).Get(0xc820f3a8d0, 0x16e4168, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/couchbase/serverjenkins/workspace/watson-unix/godeps/src/github.com/blevesearch/bleve/index/store/moss/lower.go:319 +0x168 fp=0xc8209cb220 sp=0xc8209cb158
github.com/couchbase/moss.(*snapshotWrapper).Get(0xc82072c400, 0x16e4168, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/couchbase/serverjenkins/workspace/watson-unix/godeps/src/github.com/couchbase/moss/wrap.go:61 +0x8c fp=0xc8209cb280 sp=0xc8209cb220
github.com/couchbase/moss.(*segmentStack).get(0xc8210c0c00, 0x16e4168, 0x1, 0x1, 0xffffffffffffffff, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/couchbase/serverjenkins/workspace/watson-unix/godeps/src/github.com/couchbase/moss/segment_stack.go:96 +0x396 fp=0xc8209cb350 sp=0xc8209cb280
github.com/couchbase/moss.(*segmentStack).Get(0xc8210c0c00, 0x16e4168, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/couchbase/serverjenkins/workspace/watson-unix/godeps/src/github.com/couchbase/moss/segment_stack.go:67 +0x7b fp=0xc8209cb3a8 sp=0xc8209cb350
github.com/blevesearch/bleve/index/store/moss.(*Reader).Get(0xc82072c520, 0x16e4168, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/couchbase/serverjenkins/workspace/watson-unix/godeps/src/github.com/blevesearch/bleve/index/store/moss/reader.go:26 +0x9b fp=0xc8209cb420 sp=0xc8209cb3a8
github.com/blevesearch/bleve/index/store/metrics.(*Reader).Get.func1()
/home/couchbase/serverjenkins/workspace/watson-unix/godeps/src/github.com/blevesearch/bleve/index/store/metrics/reader.go:12 +0x84 fp=0xc8209cb4c8 sp=0xc8209cb420
github.com/rcrowley/go-metrics.(*StandardTimer).Time(0xc820f3a9f0, 0xc8210c0c40)
/home/couchbase/serverjenkins/workspace/watson-unix/godeps/src/github.com/rcrowley/go-metrics/timer.go:212 +0x3d fp=0xc8209cb508 sp=0xc8209cb4c8
github.com/blevesearch/bleve/index/store/metrics.(*Reader).Get(0xc82072c540, 0x16e4168, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/couchbase/serverjenkins/workspace/watson-unix/godeps/src/github.com/blevesearch/bleve/index/store/metrics/reader.go:16 +0x13b fp=0xc8209cb548 sp=0xc8209cb508
github.com/blevesearch/bleve/index/upside_down.(*UpsideDownCouch).Open(0xc8202f6d80, 0x0, 0x0)
/home/couchbase/serverjenkins/workspace/watson-unix/godeps/src/github.com/blevesearch/bleve/index/upside_down/upside_down.go:325 +0x3b5 fp=0xc8209cb660 sp=0xc8209cb548
github.com/blevesearch/bleve.openIndexUsing(0xc8204bd1a0, 0x5b, 0x0, 0xc8201a1110, 0x0, 0x0)
/home/couchbase/serverjenkins/workspace/watson-unix/godeps/src/github.com/blevesearch/bleve/index_impl.go:201 +0x791 fp=0xc8209cb840 sp=0xc8209cb660
github.com/blevesearch/bleve.Open(0xc8204bd1a0, 0x5b, 0x0, 0x0, 0x0, 0x0)
/home/couchbase/serverjenkins/workspace/watson-unix/godeps/src/github.com/blevesearch/bleve/index.go:243 +0x50 fp=0xc8209cb890 sp=0xc8209cb840
github.com/couchbase/cbft.OpenBlevePIndexImpl(0xc820334f60, 0xe, 0xc8204bd1a0, 0x5b, 0xc8207f0540, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/home/couchbase/serverjenkins/workspace/watson-unix/goproj/src/github.com/couchbase/cbft/pindex_bleve.go:283 +0x469 fp=0xc8209cb970 sp=0xc8209cb890
github.com/couchbase/cbgt.OpenPIndexImpl(0xc820334f60, 0xe, 0xc8204bd1a0, 0x5b, 0xc8207f0540, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/home/couchbase/serverjenkins/workspace/watson-unix/goproj/src/github.com/couchbase/cbgt/pindex_impl.go:130 +0x114 fp=0xc8209cba28 sp=0xc8209cb970
github.com/couchbase/cbgt.OpenPIndex(0xc820078000, 0xc8204bd1a0, 0x5b, 0x1, 0x0, 0x0)
/home/couchbase/serverjenkins/workspace/watson-unix/goproj/src/github.com/couchbase/cbgt/pindex.go:168 +0x5af fp=0xc8209cbbd8 sp=0xc8209cba28
github.com/couchbase/cbgt.(*Manager).LoadDataDir(0xc820078000, 0x0, 0x0)
/home/couchbase/serverjenkins/workspace/watson-unix/goproj/src/github.com/couchbase/cbgt/manager.go:417 +0x693 fp=0xc8209cbe18 sp=0xc8209cbbd8
github.com/couchbase/cbgt.(*Manager).JanitorLoop(0xc820078000)
/home/couchbase/serverjenkins/workspace/watson-unix/goproj/src/github.com/couchbase/cbgt/manager_janitor.go:99 +0x92a fp=0xc8209cbf88 sp=0xc8209cbe18
runtime.goexit()
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/runtime/asm_amd64.s:1721 +0x1 fp=0xc8209cbf90 sp=0xc8209cbf88
created by github.com/couchbase/cbgt.(*Manager).Start
/home/couchbase/serverjenkins/workspace/watson-unix/goproj/src/github.com/couchbase/cbgt/manager.go:220 +0x229

goroutine 1 [chan receive]:
main.main()
/home/couchbase/serverjenkins/workspace/watson-unix/goproj/src/github.com/couchbase/cbft/cmd/cbft/main.go:240 +0x217f

goroutine 17 [syscall, locked to thread]:
runtime.goexit()
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/runtime/asm_amd64.s:1721 +0x1

goroutine 34 [select, locked to thread]:
runtime.gopark(0x11bbde0, 0xc82002ff28, 0x1025ea0, 0x6, 0x43b218, 0x2)
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/runtime/proc.go:185 +0x163
runtime.selectgoImpl(0xc82002ff28, 0x0, 0x18)
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/runtime/select.go:392 +0xa64
runtime.selectgo(0xc82002ff28)
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/runtime/select.go:212 +0x12
runtime.ensureSigM.func1()
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/runtime/signal1_unix.go:227 +0x353
runtime.goexit()
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/runtime/asm_amd64.s:1721 +0x1

goroutine 6 [select]:
github.com/blevesearch/bleve/index.AnalysisWorker(0xc820020240, 0xc8200202a0)
/home/couchbase/serverjenkins/workspace/watson-unix/godeps/src/github.com/blevesearch/bleve/index/analysis.go:70 +0x14d
created by github.com/blevesearch/bleve/index.NewAnalysisQueue
/home/couchbase/serverjenkins/workspace/watson-unix/godeps/src/github.com/blevesearch/bleve/index/analysis.go:62 +0xf2

goroutine 7 [select]:
github.com/blevesearch/bleve/index.AnalysisWorker(0xc820020240, 0xc8200202a0)
/home/couchbase/serverjenkins/workspace/watson-unix/godeps/src/github.com/blevesearch/bleve/index/analysis.go:70 +0x14d
created by github.com/blevesearch/bleve/index.NewAnalysisQueue
/home/couchbase/serverjenkins/workspace/watson-unix/godeps/src/github.com/blevesearch/bleve/index/analysis.go:62 +0xf2