[System Test] : Indexer OOM killed just after recovery

2019-06-26T01:36:16.879-07:00 [Info] other-1/o1_claims/Backstore#10703244967741809966:0 Plasma: logCleaner: starting... frag 33, data: 69668573, used: 104051673 log:(2840408105 - 2945286144)

2019-06-26T01:36:17.032-07:00 [Info] other-1/o1_result_rating/Backstore#14512317405896055992:0 Plasma: logCleaner: starting... frag 31, data: 80831849, used: 117335145 log:(2152690447 - 2270285824)

fatal error: runtime: out of memory

runtime stack:

runtime.throw(0x10cff6f, 0x16)

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/panic.go:566 +0x95

runtime.sysMap(0xc4a7b30000, 0x100000, 0x12a5600, 0x1b0a778)

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/mem_linux.go:219 +0x1d0

runtime.(*mheap).sysAlloc(0x1aefaa0, 0x100000, 0x1aefad8)

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/malloc.go:407 +0x37a

runtime.(*mheap).grow(0x1aefaa0, 0x1, 0x0)

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/mheap.go:726 +0x62

runtime.(*mheap).allocSpanLocked(0x1aefaa0, 0x1, 0x7fed00000001)

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/mheap.go:630 +0x4f2

runtime.(*mheap).alloc_m(0x1aefaa0, 0x1, 0x7f0000000007, 0x7fed0e934440)

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/mheap.go:515 +0xe0

runtime.(*mheap).alloc.func1()

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/mheap.go:579 +0x4b

runtime.systemstack(0x7fed13ffd408)

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/asm_amd64.s:314 +0xab

runtime.(*mheap).alloc(0x1aefaa0, 0x1, 0x10000000007, 0x415a69)

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/mheap.go:580 +0xa0

runtime.(*mcentral).grow(0x1af1080, 0x0)

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/mcentral.go:210 +0x94

runtime.(*mcentral).cacheSpan(0x1af1080, 0xc480361800)

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/mcentral.go:91 +0xfa

runtime.(*mcache).refill(0x7ff279d3ae10, 0x7, 0xc4334cef80)

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/mcache.go:121 +0xae

runtime.(*mcache).nextFree.func1()

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/malloc.go:505 +0x33

runtime.systemstack(0xc420029500)

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/asm_amd64.s:298 +0x79

runtime.mstart()

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/proc.go:1079

goroutine 1413 [running]:

runtime.systemstack_switch()

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/asm_amd64.s:252 fp=0xc42d85ebe8 sp=0xc42d85ebe0

runtime.(*mcache).nextFree(0x7ff279d3ae10, 0xc4a7b12e07, 0x0, 0xc480361800, 0xc480361800)

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/malloc.go:506 +0xbb fp=0xc42d85ec40 sp=0xc42d85ebe8

runtime.mallocgc(0x60, 0xfdb3c0, 0x1, 0xc4a7b12ee0)

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/malloc.go:658 +0x809 fp=0xc42d85ece0 sp=0xc42d85ec40

runtime.newobject(0xfdb3c0, 0xc4a7b2ff80)

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/malloc.go:785 +0x38 fp=0xc42d85ed10 sp=0xc42d85ece0

github.com/couchbase/indexing/secondary/indexer.NewMutation(0xc4274ecc00)

        goproj/src/github.com/couchbase/indexing/secondary/indexer/mutation.go:145 +0x8d fp=0xc42d85ed38 sp=0xc42d85ed10

github.com/couchbase/indexing/secondary/indexer.(*streamWorker).handleSingleKeyVersion(0xc4273b7f00, 0xc4a5a13008, 0x7, 0xc4000002c0, 0x2bf5ad812f57, 0xc4a5a69340, 0xc42d85ee03)

        goproj/src/github.com/couchbase/indexing/secondary/indexer/stream_reader.go:643 +0x22f fp=0xc42d85ee60 sp=0xc42d85ed38

github.com/couchbase/indexing/secondary/indexer.(*streamWorker).handleKeyVersions(0xc4273b7f00, 0xc4a5a13008, 0x7, 0x2c0, 0x2bf5ad812f57, 0xc4a5645a00, 0x23, 0x40, 0x3)

        goproj/src/github.com/couchbase/indexing/secondary/indexer/stream_reader.go:574 +0x8c fp=0xc42d85eeb8 sp=0xc42d85ee60

github.com/couchbase/indexing/secondary/indexer.(*streamWorker).start(0xc4273b7f00)

        goproj/src/github.com/couchbase/indexing/secondary/indexer/stream_reader.go:559 +0x18e fp=0xc42d85efa8 sp=0xc42d85eeb8

runtime.goexit()

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc42d85efb0 sp=0xc42d85efa8

created by github.com/couchbase/indexing/secondary/indexer.CreateMutationStreamReader

        goproj/src/github.com/couchbase/indexing/secondary/indexer/stream_reader.go:117 +0x692

goroutine 1 [runnable]:

github.com/couchbase/indexing/secondary/indexer.(*indexer).handleWorkerMsgs(0xc4200c2840, 0x1a6aaa0, 0xc4a425d920)

        goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:970 +0x7a1

github.com/couchbase/indexing/secondary/indexer.(*indexer).run(0xc4200c2840)

2019-06-26T01:36:20.413-07:00 [Info] Indexer started with command line: [/opt/couchbase/bin/indexer -adminPort=9100 -scanPort=9101 -httpPort=9102 -streamInitPort=9103 -streamCatchupPort=9104 -streamMaintPort=9105 --httpsPort=19102 --certFile=/opt/couchbase/var/lib/couchbase/config/memcached-cert.pem --keyFile=/opt/couchbase/var/lib/couchbase/config/memcached-key.pem -vbuckets=1024 -cluster=127.0.0.1:8091 -storageDir=/data/@2i -diagDir=/opt/couchbase/var/lib/couchbase/crash -nodeUUID=af7cd8cada6c2aaf4d8be478d865fa10 -ipv6=false -isEnterprise=true]