[System Test on cloud] Panic in Query

2023-08-04T13:31:21.504+00:00 [Info] CreateIndex 0 default2 scope_0 coll_0/idx9_5KNJYAT61R_idxprefix using:GSI exprType:N1QL whereExpr:<ud>()</ud> secExprs:<ud>([(all (array flatten_keys((`r`.`author`), ((`r`.`ratings`).`Rooms`)) for `r` in `reviews` end)) `free_parking`])</ud> desc:[false false false] indexMissingLeadingKey:false isPrimary:false scheme:SINGLE  partitionKeys:<ud>([])</ud> with:{"num_replica":1} - elapsed(3.113613ms) err(Index creation for index idx9_5KNJYAT61R_idxprefix, bucket default2, scope scope_0, collection coll_0 cannot start. Reason: Create index or Alter replica cannot proceed due to rebalance in progress..)

fatal error: runtime: out of memory

fatal error: runtime: out of memory

runtime stack:

runtime.throw({0x291f646?, 0x38002a488d?})

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/panic.go:1047 +0x5d fp=0x7f5741469c20 sp=0x7f5741469bf0 pc=0xf247dd

runtime.sysMapOS(0xc704000000, 0x400000?)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/mem_linux.go:187 +0x11b fp=0x7f5741469c68 sp=0x7f5741469c20 pc=0xf0449b

runtime.sysMap(0x44e53a0?, 0xf195da?, 0x44f5538?)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/mem.go:142 +0x35 fp=0x7f5741469c98 sp=0x7f5741469c68 pc=0xf03e75

runtime.(*mheap).grow(0x44e53a0, 0x2000?)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/mheap.go:1522 +0x252 fp=0x7f5741469d10 sp=0x7f5741469c98 pc=0xf14992

runtime.(*mheap).allocSpan(0x44e53a0, 0x1, 0x0, 0x8b?)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/mheap.go:1243 +0x1b7 fp=0x7f5741469da8 sp=0x7f5741469d10 pc=0xf140d7

runtime.(*mheap).alloc.func1()

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/mheap.go:961 +0x65 fp=0x7f5741469df0 sp=0x7f5741469da8 pc=0xf13b85

runtime.systemstack()

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/asm_amd64.s:496 +0x49 fp=0x7f5741469df8 sp=0x7f5741469df0 pc=0xf59d09

goroutine 1570733 [running]:

runtime.systemstack_switch()

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/asm_amd64.s:463 fp=0xc001d32680 sp=0xc001d32678 pc=0xf59ca0

runtime.(*mheap).alloc(0xf1e150?, 0x1b9ab85?, 0x5?)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/mheap.go:955 +0x65 fp=0xc001d326c8 sp=0xc001d32680 pc=0xf13ac5

runtime.(*mcentral).grow(0x2000?)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/mcentral.go:246 +0x57 fp=0xc001d32708 sp=0xc001d326c8 pc=0xf037d7

runtime.(*mcentral).cacheSpan(0x44f5af8)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/mcentral.go:166 +0x306 fp=0xc001d32760 sp=0xc001d32708 pc=0xf03626

runtime.(*mcache).refill(0x7f576f4c9f18, 0x7?)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/mcache.go:182 +0x152 fp=0xc001d327a0 sp=0xc001d32760 pc=0xf02d72

runtime.(*mcache).nextFree(0x7f576f4c9f18, 0x7)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/malloc.go:855 +0x85 fp=0xc001d327e8 sp=0xc001d327a0 pc=0xef8fe5

runtime.mallocgc(0x18, 0x0, 0x0)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/malloc.go:1042 +0x455 fp=0xc001d32850 sp=0xc001d327e8 pc=0xef95d5

runtime.growslice(0x44fbe40, 0x15?, 0xf8f4e?, 0xc6fe328000?, 0xc6fe324050?)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/slice.go:266 +0x4a8 fp=0xc001d328b0 sp=0xc001d32850 pc=0xf3d868

google.golang.org/protobuf/internal/impl.consumeBytes({0xc02ec610b2?, 0xf1e150?, 0xc001d32960?}, {0x18?}, 0xd8?, 0x10?, {0x0, {0x2f1e468, 0xc0004157a0}, 0x270f})

	/home/couchbase/.cbdepscache/gomodcache/pkg/mod/google.golang.org/protobuf@v1.28.0/internal/impl/codec_gen.go:5331 +0x76 fp=0xc001d32920 sp=0xc001d328b0 pc=0x1bb5276

google.golang.org/protobuf/internal/impl.(*MessageInfo).unmarshalPointer(0xc0004686e0, {0xc02ec610af?, 0x18, 0x7f4d2538dbd8?}, {0x7f576f4c9f18?}, 0x0, {0x0, {0x2f1e468, 0xc0004157a0}, 0x2710})

	/home/couchbase/.cbdepscache/gomodcache/pkg/mod/google.golang.org/protobuf@v1.28.0/internal/impl/decode.go:147 +0x3c3 fp=0xc001d32a58 sp=0xc001d32920 pc=0x1bc8963

google.golang.org/protobuf/internal/impl.(*MessageInfo).unmarshal(0xc703e7e050?, {{}, {0x2f3c850, 0xc703e7c040}, {0xc02ec610af, 0x18, 0xf8f51}, 0x0, {0x2f1e468, 0xc0004157a0}, ...})

	/home/couchbase/.cbdepscache/gomodcache/pkg/mod/google.golang.org/protobuf@v1.28.0/internal/impl/decode.go:65 +0xa5 fp=0xc001d32ab8 sp=0xc001d32a58 pc=0x1bc8565

google.golang.org/protobuf/internal/impl.(*MessageInfo).unmarshal-fm({{}, {0x2f3c850, 0xc703e7c040}, {0xc02ec610af, 0x18, 0xf8f51}, 0x0, {0x2f1e468, 0xc0004157a0}, 0x2710})

	<autogenerated>:1 +0xa5 fp=0xc001d32b18 sp=0xc001d32ab8 pc=0x1beb1c5

google.golang.org/protobuf/proto.UnmarshalOptions.unmarshal({{}, 0x1, 0x1, 0x0, {0x2f1e468, 0xc0004157a0}, 0x2710}, {0xc02ec610af, 0x18, 0xf8f51}, ...)

	/home/couchbase/.cbdepscache/gomodcache/pkg/mod/google.golang.org/protobuf@v1.28.0/proto/decode.go:104 +0x211 fp=0xc001d32bd0 sp=0xc001d32b18 pc=0x1b62391

google.golang.org/protobuf/proto.UnmarshalOptions.UnmarshalState({{}, 0x1, 0x1, 0x0, {0x2f1e468, 0xc0004157a0}, 0x2710}, {{}, {0x2f3c850, 0xc703e7c040}, ...})

https://cb-engineering.s3.amazonaws.com/VolTestAug3/collectinfo-2023-08-05T153149-ns_1%40svc-d-node-001.vtp2vfojktbbff7q.sandbox.nonprod-project-avengers.com.zip

https://cb-engineering.s3.amazonaws.com/VolTestAug3/collectinfo-2023-08-05T153149-ns_1%40svc-d-node-002.vtp2vfojktbbff7q.sandbox.nonprod-project-avengers.com.zip

https://cb-engineering.s3.amazonaws.com/VolTestAug3/collectinfo-2023-08-05T153149-ns_1%40svc-d-node-003.vtp2vfojktbbff7q.sandbox.nonprod-project-avengers.com.zip

https://cb-engineering.s3.amazonaws.com/VolTestAug3/collectinfo-2023-08-05T153149-ns_1%40svc-d-node-008.vtp2vfojktbbff7q.sandbox.nonprod-project-avengers.com.zip

https://cb-engineering.s3.amazonaws.com/VolTestAug3/collectinfo-2023-08-05T153149-ns_1%40svc-i-node-004.vtp2vfojktbbff7q.sandbox.nonprod-project-avengers.com.zip

https://cb-engineering.s3.amazonaws.com/VolTestAug3/collectinfo-2023-08-05T153149-ns_1%40svc-i-node-005.vtp2vfojktbbff7q.sandbox.nonprod-project-avengers.com.zip

https://cb-engineering.s3.amazonaws.com/VolTestAug3/collectinfo-2023-08-05T153149-ns_1%40svc-i-node-009.vtp2vfojktbbff7q.sandbox.nonprod-project-avengers.com.zip

https://cb-engineering.s3.amazonaws.com/VolTestAug3/collectinfo-2023-08-05T153149-ns_1%40svc-q-node-006.vtp2vfojktbbff7q.sandbox.nonprod-project-avengers.com.zip

https://cb-engineering.s3.amazonaws.com/VolTestAug3/collectinfo-2023-08-05T153149-ns_1%40svc-q-node-007.vtp2vfojktbbff7q.sandbox.nonprod-project-avengers.com.zip