Plasma: SMR reclaim pending is higher than expected , seeing OOM

2023-01-10T12:00:36.306+00:00 [Info] bucket-1/index19-5/Backstore#5691476633885533671:0 Plasma: SMR reclaim pending is higher than expected: pending = 160 KB (expected = 103 KB), wCtxCnt = 17, objCnt 16, changed reclaimList flush threshold from 2 to 0, changed reclaimSize flush threshold from 4 KB to 4 KB.

2023-01-10T12:00:36.318+00:00 [Info] bucket-1/index34-10/Backstore#11289282397930634741:0 Plasma: SMR reclaim pending is higher than expected: pending = 1351 KB (expected = 103 KB), wCtxCnt = 19, objCnt 7, changed reclaimList flush threshold from 0 to 0, changed reclaimSize flush threshold from 5 KB to 4 KB.

2023-01-10T12:00:36.329+00:00 [Info] bucket-1/index41-10/Backstore#11044061330604066900:0 Plasma: SMR reclaim pending is higher than expected: pending = 157 KB (expected = 103 KB), wCtxCnt = 24, objCnt 4, changed reclaimList flush threshold from 4 to 0, changed reclaimSize flush threshold from 3 KB to 3 KB.

2023-01-10T12:00:36.905+00:00 [Info] bucket-1/index77-10/Mainstore#7401733339373058963:0 Plasma: SMR reclaim pending is higher than expected: pending = 283 KB (expected = 103 KB), wCtxCnt = 13, objCnt 6, changed reclaimList flush threshold from 0 to 0, changed reclaimSize flush threshold from 7 KB to 6 KB.

2023-01-10T12:00:37.250+00:00 [Info] bucket-1/index58-10/Mainstore#16946451062143317954:0 Plasma: SMR reclaim pending is higher than expected: pending = 148 KB (expected = 103 KB), wCtxCnt = 11, objCnt 8, changed reclaimList flush threshold from 0 to 0, changed reclaimSize flush threshold from 8 KB to 7 KB.

fatal error: runtime: out of memory

runtime stack:

runtime.throw({0x1236d36?, 0x400000?})

    /home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.5/go/src/runtime/panic.go:992 +0x71

runtime.sysMap(0xc122000000, 0x0?, 0x1?)

    /home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.5/go/src/runtime/mem_linux.go:189 +0x11b

runtime.(*mheap).grow(0x2026220, 0x46f181?)

    /home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.5/go/src/runtime/mheap.go:1404 +0x225

runtime.(*mheap).allocSpan(0x2026220, 0x1, 0x0, 0xc)

    /home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.5/go/src/runtime/mheap.go:1170 +0x171

runtime.(*mheap).alloc.func1()

    /home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.5/go/src/runtime/mheap.go:912 +0x65

runtime.systemstack()

    /home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.5/go/src/runtime/asm_amd64.s:469 +0x49goroutine 90094 [running]:

runtime.systemstack_switch()

    /home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.5/go/src/runtime/asm_amd64.s:436 fp=0xc017071360 sp=0xc017071358 pc=0x46cf60

runtime.(*mheap).alloc(0x7fd509651c00?, 0xe8ddd2?, 0x10?)

    /home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.5/go/src/runtime/mheap.go:906 +0x65 fp=0xc0170713a8 sp=0xc017071360 pc=0x42d785

runtime.(*mcentral).grow(0x0?)

    /home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.5/go/src/runtime/mcentral.go:244 +0x5b fp=0xc0170713f0 sp=0xc0170713a8 pc=0x41e51b

runtime.(*mcentral).cacheSpan(0x2036d60)

    /home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.5/go/src/runtime/mcentral.go:164 +0x30f fp=0xc017071448 sp=0xc0170713f0 pc=0x41e34f

runtime.(*mcache).refill(0x7fecb85a0b38, 0xc)

    /home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.5/go/src/runtime/mcache.go:162 +0xaf fp=0xc017071480 sp=0xc017071448 pc=0x41d9cf

runtime.(*mcache).nextFree(0x7fecb85a0b38, 0xc)

    /home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.5/go/src/runtime/malloc.go:886 +0x85 fp=0xc0170714c8 sp=0xc017071480 pc=0x4138a5

runtime.mallocgc(0x40, 0x11808c0, 0x1)

    /home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.5/go/src/runtime/malloc.go:1085 +0x4e5 fp=0xc017071540 sp=0xc0170714c8 pc=0x413f25

runtime.newobject(0xc0170716b0?)

    /home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.5/go/src/runtime/malloc.go:1259 +0x27 fp=0xc017071568 sp=0xc017071540 pc=0x414467

github.com/couchbase/plasma.newPage2(...)

    /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/page.go:1902

github.com/couchbase/plasma.newPage(...)

    /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/page.go:1897

github.com/couchbase/plasma.(*Plasma).ReadPage(0x7fd509651f00?, {0xc017071628?}, 0x0, 0xc005e5aa20)

    /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/pagetable.go:207 +0x87 fp=0xc0170715d8 sp=0xc017071568 pc=0xce1347

github.com/couchbase/plasma.(*Plasma).Persist(0xc00b744000, {0xc011ed46e0?}, 0x1, 0xc005e5aa20)

    /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/persistor.go:158 +0x266 fp=0xc017071858 sp=0xc0170715d8 pc=0xce1e86

github.com/couchbase/plasma.(*Plasma).tryEvictPagesOnce(0xc00b744000, 0xc005e5aa20, 0x1, 0x1, 0x0?)

    /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/swapper.go:294 +0x20c fp=0xc0170718d0 sp=0xc017071858 pc=0xd1cbcc

github.com/couchbase/plasma.(*Plasma).tryEvictPages(0xc00b744000, 0x18?)

    /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/swapper.go:257 +0x72 fp=0xc017071918 sp=0xc0170718d0 pc=0xd1c912

github.com/couchbase/plasma.(*wCtx).tryThrottleForMemory(0xc005e5aa20, 0xc0170719a8?, 0x1?, 0x0)

    /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/plasma.go:1747 +0x130 fp=0xc017071988 sp=0xc017071918 pc=0xced810

github.com/couchbase/plasma.(*Writer).Begin(0xc00c38aa00)

    /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/writer.go:44 +0x8c fp=0xc0170719b8 sp=0xc017071988 pc=0xd28eac

github.com/couchbase/indexing/secondary/indexer.(*plasmaSlice).deleteSecIndex(0xc00a344580, {0xc057b413b0, 0x10, 0x10}, {0xc07ab2c670, 0xb, 0x0?}, 0x7)

    /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:1227 +0xb6 fp=0xc017071b28 sp=0xc0170719b8 pc=0xe92696

github.com/couchbase/indexing/secondary/indexer.(*plasmaSlice).insertSecIndex(0xc00a344580, {0xc07ab2c670, 0xb, 0x10}, {0xc057b413b0, 0x10, 0x10}, 0x7, 0x0, 0xc08a880c00)

    /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:849 +0x192 fp=0xc017071dd0 sp=0xc017071b28 pc=0xe8ddd2

github.com/couchbase/indexing/secondary/indexer.(*plasmaSlice).insert(0xc00a344580, {0xc07ab2c670?, 0x0?, 0x2?}, {0xc057b413b0?, 0x3231333339393932?, 0x69222c3839393230?}, 0x7365636e6174736e?, 0x22?, 0xc08a880c00)

    /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:807 +0xb9 fp=0xc017071e60 sp=0xc017071dd0 pc=0xe8d759

github.com/couchbase/indexing/secondary/indexer.(*plasmaSlice).handleCommandsWorker(0xc00a344580, 0x7)

    /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:680 +0x425 fp=0xc017071fc0 sp=0xc017071e60 pc=0xe8c645

github.com/couchbase/indexing/secondary/indexer.(*plasmaSlice).initWriters.func1()

    /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:3199 +0x2a fp=0xc017071fe0 sp=0xc017071fc0 pc=0xea498a

runtime.goexit()

    /home/couchbase/.cbdepscache/exploded/x86_64/go-1.18.5/go/src/runtime/asm_amd64.s:1571 +0x1 fp=0xc017071fe8 sp=0xc017071fe0 pc=0x46f181

created by github.com/couchbase/indexing/secondary/indexer.(*plasmaSlice).initWriters

    /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:3199 +0x3cdgoroutine 1 [select]:

github.com/couchbase/indexing/secondary/indexer.(*indexer).run(0xc000190000)

    /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:1065 +0x1f9

github.com/couchbase/indexing/secondary/indexer.NewIndexer(0x1ffe580?)

    /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/indexer.go:511 +0x262c

main.main()

    goproj/src/github.com/couchbase/indexing/secondary/cmd/indexer/main.go:182 +0xfaegoroutine 18 [select, 84 minutes]:

github.com/couchbase/cbauth/cbauthimpl.(*tlsNotifier).loop(0xc00020c0f0)

    /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/cbauthimpl/impl.go:389 +0x67

created by github.com/couchbase/cbauth/cbauthimpl.NewSVCForTest

    /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/cbauthimpl/impl.go:550 +0x37agoroutine 19 [select, 84 minutes]:

github.com/couchbase/cbauth/cbauthimpl.(*cfgChangeNotifier).loop(0xc00020c108)

    /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/cbauthimpl/impl.go:309 +0x85

created by github.com/couchbase/cbauth/cbauthimpl.NewSVCForTest

    /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/cbauthimpl/impl.go:551 +0x3ca