[CI] [FDB] SIGSEGV: segmentation violation, signal arrived during cgo execution

=== RUN   TestThreeIndexCreates

2024/03/12 23:46:54 In TestThreeIndexCreates()

2024/03/12 23:46:58 Created the secondary index index_balance. Waiting for it become active

2024/03/12 23:46:58 Index is 7424671226615783676 now active

2024/03/12 23:47:03 Created the secondary index index_email. Waiting for it become active

2024/03/12 23:47:03 Index is 4303538726951954908 now active

2024/03/12 23:47:03 Create docs mutations

2024/03/12 23:47:03 Using n1ql client

2024-03-12T23:47:04.181+05:30 [Error] WatcherServer.runOnce() : Watcher terminated unexpectedly.

2024-03-12T23:47:04.184+05:30 [Error] [GsiScanClient:"127.0.0.1:9107"] GsiScanClient::streamResponse req(7184310668216595291) connection "127.0.0.1:51998" closed `EOF` 

2024-03-12T23:47:04.184+05:30 [Error] [GsiScanClient:"127.0.0.1:9107"] Range(7184310668216595291) response failed `EOF`

2024-03-12T23:47:04.184+05:30 [Error] metadataClient:PickRandom: Fail to find indexer for all index partitions. Num partition 1.  Partition with instances 0 

2024-03-12T23:47:04.184+05:30 [Error] metadataClient:PickRandom: Replicas - [16468395304131228493], PrunedReplica - map[], FilteredReplica map[]

2024-03-12T23:47:04.184+05:30 [Error] WatcherServer.runOnce() : Watcher terminated unexpectedly.

2024-03-12T23:47:04.196+05:30 [Error] metadataClient:PickRandom: Fail to find indexer for all index partitions. Num partition 1.  Partition with instances 0 

2024-03-12T23:47:04.196+05:30 [Error] metadataClient:PickRandom: Replicas - [16468395304131228493], PrunedReplica - map[], FilteredReplica map[]

Scan error:  dial tcp 127.0.0.1:9107: connect: connection refused from [127.0.0.1:9107] - cause:  dial tcp 127.0.0.1:9107: connect: connection refused from [127.0.0.1:9107]

    common_test.go:173: Error in scan:  dial tcp 127.0.0.1:9107: connect: connection refused from [127.0.0.1:9107]

--- FAIL: TestThreeIndexCreates (9.43s)

2024-03-12T23:47:03.704+05:30 [Info] Timekeeper::handleMergeStreamAck StreamId INIT_STREAM KeyspaceId default

SIGSEGV: segmentation violation

PC=0x7f23f861727f m=4 sigcode=128

signal arrived during cgo execution

goroutine 7270 [syscall]:

runtime.cgocall(0x164f6d2, 0xc00a6939f0)

    /home/buildbot/.cbdepscache/exploded/x86_64/go-1.21.3/go/src/runtime/cgocall.go:157 +0x4b fp=0xc00a6939c8 sp=0xc00a693990 pc=0x40dfcb

github.com/couchbase/indexing/secondary/fdb._Cfunc_fdb_get_kv(0x7f2397205a00, 0xc00016dad0, 0x28, 0xc006f17e08, 0xc0065f4f00)

    _cgo_gotypes.go:763 +0x4b fp=0xc00a6939f0 sp=0xc00a6939c8 pc=0xc9fb2b

github.com/couchbase/indexing/secondary/fdb.(*KVStore).GetKV.func1(0xc004745860?, 0xc00a693ab8?, 0xa693ab8?, 0xa693ab80172c860?, 0x0?)

    /opt/build/goproj/src/github.com/couchbase/indexing/secondary/fdb/kv.go:35 +0xb1 fp=0xc00a693a58 sp=0xc00a6939f0 pc=0xcace91

github.com/couchbase/indexing/secondary/fdb.(*KVStore).GetKV(0xc004745860, {0xc00016dad0, 0x28, 0xc005a37bf0?})

    /opt/build/goproj/src/github.com/couchbase/indexing/secondary/fdb/kv.go:35 +0xfd fp=0xc00a693b08 sp=0xc00a693a58 pc=0xcacb7d

github.com/couchbase/indexing/secondary/indexer.(*fdbSlice).getBackIndexEntry(0xc000100c00, {0xc00016dad0, 0x28, 0x30}, 0x0)

    /opt/build/goproj/src/github.com/couchbase/indexing/secondary/indexer/forestdb_slice_writer.go:936 +0x96 fp=0xc00a693b88 sp=0xc00a693b08 pc=0x13f1d36

github.com/couchbase/indexing/secondary/indexer.(*fdbSlice).insertSecIndex(0xc000100c00, {0xc0086ca910, 0x48, 0x50}, {0xc00016dad0, 0x28, 0x30}, 0x0)

    /opt/build/goproj/src/github.com/couchbase/indexing/secondary/indexer/forestdb_slice_writer.go:462 +0x6e fp=0xc00a693de0 sp=0xc00a693b88 pc=0x13ec4ce

github.com/couchbase/indexing/secondary/indexer.(*fdbSlice).insert(0xc000100c00, {0xc0086ca910?, 0x1?, 0x50?}, {0xc008bf7a20?, 0x1a?, 0x20?}, {0xc00016dad0, 0x28, 0x30}, ...)

    /opt/build/goproj/src/github.com/couchbase/indexing/secondary/indexer/forestdb_slice_writer.go:413 +0xcf fp=0xc00a693e78 sp=0xc00a693de0 pc=0x13ebb0f

github.com/couchbase/indexing/secondary/indexer.(*fdbSlice).handleCommandsWorker(0xc000100c00, 0x0)

    /opt/build/goproj/src/github.com/couchbase/indexing/secondary/indexer/forestdb_slice_writer.go:359 +0x268 fp=0xc00a693fc0 sp=0xc00a693e78 pc=0x13eb3e8

github.com/couchbase/indexing/secondary/indexer.NewForestDBSlice.func1()

    /opt/build/goproj/src/github.com/couchbase/indexing/secondary/indexer/forestdb_slice_writer.go:170 +0x25 fp=0xc00a693fe0 sp=0xc00a693fc0 pc=0x13ea945

runtime.goexit()

    /home/buildbot/.cbdepscache/exploded/x86_64/go-1.21.3/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00a693fe8 sp=0xc00a693fe0 pc=0x478361

created by github.com/couchbase/indexing/secondary/indexer.NewForestDBSlice in goroutine 1

    /opt/build/goproj/src/github.com/couchbase/indexing/secondary/indexer/forestdb_slice_writer.go:170 +0xe93

goroutine 184 [runnable]:

runtime.cgocall(0x164f73e, 0xc005b5cab8)

    /home/buildbot/.cbdepscache/exploded/x86_64/go-1.21.3/go/src/runtime/cgocall.go:157 +0x4b fp=0xc005b5ca90 sp=0xc005b5ca58 pc=0x40dfcb

github.com/couchbase/indexing/secondary/fdb._Cfunc_fdb_set_kv(0x7f23972046c0, 0xc00733b470, 0x28, 0xc009c4f840, 0x34)

    _cgo_gotypes.go:1082 +0x4b fp=0xc005b5cab8 sp=0xc005b5ca90 pc=0xca0deb

github.com/couchbase/indexing/secondary/fdb.(*KVStore).SetKV.func1(0xc004d93b30?, 0x30aaaef4f0?, 0x2ed4880?, 0x0?, 0x0?)

    /opt/build/goproj/src/github.com/couchbase/indexing/secondary/fdb/kv.go:64 +0xa7 fp=0xc005b5cb20 sp=0xc005b5cab8 pc=0xcad187

github.com/couchbase/indexing/secondary/fdb.(*KVStore).SetKV(0xc004d93b30, {0xc00733b470, 0x28, 0x30?}, {0xc009c4f840, 0x34, 0x0?})

    /opt/build/goproj/src/github.com/couchbase/indexing/secondary/fdb/kv.go:64 +0xac fp=0xc005b5cb88 sp=0xc005b5cb20 pc=0xcad00c

github.com/couchbase/indexing/secondary/indexer.(*fdbSlice).insertSecIndex(0xc004da6400, {0xc009c4f840, 0x34, 0x40}, {0xc00733b470, 0x28, 0x30}, 0x0)

    /opt/build/goproj/src/github.com/couchbase/indexing/secondary/indexer/forestdb_slice_writer.go:513 +0x765 fp=0xc005b5cde0 sp=0xc005b5cb88 pc=0x13ecbc5

github.com/couchbase/indexing/secondary/indexer.(*fdbSlice).insert(0xc004da6400, {0xc009c4f840?, 0x1?, 0x40?}, {0xc006593d20?, 0x9?, 0x10?}, {0xc00733b470, 0x28, 0x30}, ...)

    /opt/build/goproj/src/github.com/couchbase/indexing/secondary/indexer/forestdb_slice_writer.go:413 +0xcf fp=0xc005b5ce78 sp=0xc005b5cde0 pc=0x13ebb0f

github.com/couchbase/indexing/secondary/indexer.(*fdbSlice).handleCommandsWorker(0xc004da6400, 0x0)

    /opt/build/goproj/src/github.com/couchbase/indexing/secondary/indexer/forestdb_slice_writer.go:359 +0x268 fp=0xc005b5cfc0 sp=0xc005b5ce78 pc=0x13eb3e8

github.com/couchbase/indexing/secondary/indexer.NewForestDBSlice.func1()

    /opt/build/goproj/src/github.com/couchbase/indexing/secondary/indexer/forestdb_slice_writer.go:170 +0x25 fp=0xc005b5cfe0 sp=0xc005b5cfc0 pc=0x13ea945

runtime.goexit()

    /home/buildbot/.cbdepscache/exploded/x86_64/go-1.21.3/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc005b5cfe8 sp=0xc005b5cfe0 pc=0x478361

created by github.com/couchbase/indexing/secondary/indexer.NewForestDBSlice in goroutine 144

    /opt/build/goproj/src/github.com/couchbase/indexing/secondary/indexer/forestdb_slice_writer.go:170 +0xe93

goroutine 3345 [runnable]:

runtime.cgocall(0x164f6d2, 0xc004ce79f0)

    /home/buildbot/.cbdepscache/exploded/x86_64/go-1.21.3/go/src/runtime/cgocall.go:157 +0x4b fp=0xc004ce79c8 sp=0xc004ce7990 pc=0x40dfcb

github.com/couchbase/indexing/secondary/fdb._Cfunc_fdb_get_kv(0x7f23972069c0, 0xc00b67f9e0, 0x28, 0xc006b39978, 0xc0069089c8)

    _cgo_gotypes.go:763 +0x4b fp=0xc004ce79f0 sp=0xc004ce79c8 pc=0xc9fb2b

github.com/couchbase/indexing/secondary/fdb.(*KVStore).GetKV.func1(0xc005ff3c50?, 0xc004ce7ab8?, 0x4ce7ab8?, 0x4ce7ab80172c860?, 0x0?)

    /opt/build/goproj/src/github.com/couchbase/indexing/secondary/fdb/kv.go:35 +0xb1 fp=0xc004ce7a58 sp=0xc004ce79f0 pc=0xcace91

github.com/couchbase/indexing/secondary/fdb.(*KVStore).GetKV(0xc005ff3c50, {0xc00b67f9e0, 0x28, 0xc004f328d0?})

    /opt/build/goproj/src/github.com/couchbase/indexing/secondary/fdb/kv.go:35 +0xfd fp=0xc004ce7b08 sp=0xc004ce7a58 pc=0xcacb7d

github.com/couchbase/indexing/secondary/indexer.(*fdbSlice).getBackIndexEntry(0xc004cc8800, {0xc00b67f9e0, 0x28, 0x30}, 0x0)

    /opt/build/goproj/src/github.com/couchbase/indexing/secondary/indexer/forestdb_slice_writer.go:936 +0x96 fp=0xc004ce7b88 sp=0xc004ce7b08 pc=0x13f1d36

github.com/couchbase/indexing/secondary/indexer.(*fdbSlice).insertSecIndex(0xc004cc8800, {0xc009c4fd40, 0x38, 0x40}, {0xc00b67f9e0, 0x28, 0x30}, 0x0)

    /opt/build/goproj/src/github.com/couchbase/indexing/secondary/indexer/forestdb_slice_writer.go:462 +0x6e fp=0xc004ce7de0 sp=0xc004ce7b88 pc=0x13ec4ce

github.com/couchbase/indexing/secondary/indexer.(*fdbSlice).insert(0xc004cc8800, {0xc009c4fd40?, 0x1?, 0x40?}, {0xc004dc77a0?, 0xe?, 0x10?}, {0xc00b67f9e0, 0x28, 0x30}, ...)

    /opt/build/goproj/src/github.com/couchbase/indexing/secondary/indexer/forestdb_slice_writer.go:413 +0xcf fp=0xc004ce7e78 sp=0xc004ce7de0 pc=0x13ebb0f

github.com/couchbase/indexing/secondary/indexer.(*fdbSlice).handleCommandsWorker(0xc004cc8800, 0x0)

    /opt/build/goproj/src/github.com/couchbase/indexing/secondary/indexer/forestdb_slice_writer.go:359 +0x268 fp=0xc004ce7fc0 sp=0xc004ce7e78 pc=0x13eb3e8

github.com/couchbase/indexing/secondary/indexer.NewForestDBSlice.func1()

    /opt/build/goproj/src/github.com/couchbase/indexing/secondary/indexer/forestdb_slice_writer.go:170 +0x25 fp=0xc004ce7fe0 sp=0xc004ce7fc0 pc=0x13ea945

runtime.goexit()

    /home/buildbot/.cbdepscache/exploded/x86_64/go-1.21.3/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc004ce7fe8 sp=0xc004ce7fe0 pc=0x478361

created by github.com/couchbase/indexing/secondary/indexer.NewForestDBSlice in goroutine 1

    /opt/build/goproj/src/github.com/couchbase/indexing/secondary/indexer/forestdb_slice_writer.go:170 +0xe93