[System Test] Fatal error seen while dropping indexes : plasmaSnapshot::Iterate: panic detected while iterating snapshot low key

2021-05-30T01:09:27.268-07:00 [Info] lifecycleMgr.dispatchRequest: op OPCODE_DROP_INDEX elapsed 368.674609ms len(expediates) 0 len(incomings) 0 len(outgoings) 0 error <nil>

2021-05-30T01:09:27.272-07:00 [Info] bucket1/idx2_SW2Oc/Mainstore#8283286452070396845:0 Plasma: Plasma.Close(): Done persist all page, concurrency [8]

2021-05-30T01:09:27.272-07:00 [Info] Shard /data/@2i/shards/shard4(4) : Shard.closeInstance: Closing instance [/data/@2i/bucket1_idx2_SW2Oc_8283286452070396845_0.index/mainIndex]

2021-05-30T01:09:27.272-07:00 [Info] Shard /data/@2i/shards/shard4(4) : Shard.closeInstance: instance /data/@2i/bucket1_idx2_SW2Oc_8283286452070396845_0.index/mainIndex closed

2021-05-30T01:09:27.295-07:00 [Fatal] plasmaSnapshot::Iterate: panic detected while iterating snapshot low key = <ud>()</ud> high key = <ud>()</ud> Index idx2_SW2Oc, Bucket bucket1, IndexInstId 8283286452070396845, PartitionId 0

2021-05-30T01:09:27.318-07:00 [Info] bucket1/idx2_SW2Oc/Mainstore#8283286452070396845:0 Plasma: Plasma.Close: Done closing plasma instance

2021-05-30T01:09:27.318-07:00 [Info] bucket1/idx2_SW2Oc/Backstore#8283286452070396845:0 Plasma: Plasma.Close: Begin closing plasma instance

2021-05-30T01:09:27.318-07:00 [Info] bucket1/idx2_SW2Oc/Backstore#8283286452070396845:0 Plasma: Plasma.monitor: Received stop message

2021-05-30T01:09:27.318-07:00 [Info] bucket1/idx2_SW2Oc/Backstore#8283286452070396845:0 Plasma: Plasma.monitor: Stopped monitor

2021-05-30T01:09:27.318-07:00 [Info] Shard /data/@2i/shards/shard3(3) : Shard.StopInstance: Stopping instance [/data/@2i/bucket1_idx2_SW2Oc_8283286452070396845_0.index/docIndex]

2021-05-30T01:09:27.318-07:00 [Info] Shard /data/@2i/shards/shard3(3) : Shard.removeFromSwapper: Begin removing from swapper instance [/data/@2i/shards/shard3]

2021-05-30T01:09:27.318-07:00 [Info] Shard /data/@2i/shards/shard3(3) : swapperDaemon.removeInstance: Begin removing instance [/data/@2i/shards/shard3]

2021-05-30T01:09:27.321-07:00 [Fatal] goroutine 131704007 [running]:

github.com/couchbase/indexing/secondary/logging.(*destination).StackTraceAll(...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/logging/logging.go:175

github.com/couchbase/indexing/secondary/logging.StackTraceAll(0x215a4e0, 0x2180001)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/logging/logging.go:317 +0x6d

github.com/couchbase/indexing/secondary/indexer.(*plasmaSnapshot).Iterate.func1(0x14f3680, 0xc01ae239e0, 0x14f3680, 0xc01ae23a20, 0xc00a6ab450)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:2779 +0x3b9

panic(0x10e7000, 0x20e2790)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/runtime/panic.go:679 +0x1b2

github.com/couchbase/nitro/skiplist.(*Skiplist).GetAccesBarrier(...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/nitro/skiplist/skiplist.go:126

github.com/couchbase/plasma.(*wCtx).EndSMR(...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/smr.go:107

github.com/couchbase/plasma.(*wCtx).EndTx(0xc00e1facf0, 0xc02588dfc0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/smr.go:89 +0x45

github.com/couchbase/plasma.(*MVCCIterator).Close(0xc0238a89c0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/mvcc.go:258 +0x7b

github.com/couchbase/indexing/secondary/indexer.(*plasmaSnapshot).Iterate(0xc00a6ab450, 0x14ec0c0, 0xc01fe3d8a0, 0x14f3680, 0xc01ae239e0, 0x14f3680, 0xc01ae23a20, 0x3, 0x1327118, 0xc00e8f2450, ...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:2849 +0x4f7

github.com/couchbase/indexing/secondary/indexer.(*plasmaSnapshot).Range(0xc00a6ab450, 0x14ec0c0, 0xc01fe3d8a0, 0x14f3680, 0xc01ae239e0, 0x14f3680, 0xc01ae23a20, 0x3, 0xc00e8f2450, 0x10f0ec0, ...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:2763 +0xb8

github.com/couchbase/indexing/secondary/indexer.scanSingleSlice(0xc008a3e680, 0x14f3680, 0xc01ae239e0, 0x14f3680, 0xc01ae23a20, 0x3, 0x12b4051, 0xb, 0xc01f829590, 0x1, ...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/scan_scatter.go:205 +0x233

github.com/couchbase/indexing/secondary/indexer.scanOne(0xc008a3e680, 0x14f3680, 0xc01ae239e0, 0x14f3680, 0xc01ae23a20, 0x3, 0x12b4051, 0xb, 0xc01f829590, 0x1, ...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/scan_scatter.go:147 +0x111

github.com/couchbase/indexing/secondary/indexer.scatter(0xc008a3e680, 0x14f3680, 0xc01ae239e0, 0x14f3680, 0xc01ae23a20, 0x3, 0x12b4051, 0xb, 0xc01f829590, 0x1, ...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/scan_scatter.go:54 +0xc4

github.com/couchbase/indexing/secondary/indexer.(*IndexScanSource).Routine(0xc01605b570, 0x0, 0x0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/scan_pipeline.go:339 +0xdae

github.com/couchbase/indexing/secondary/pipeline.(*Pipeline).runIt.func1(0xc01ae23a60, 0xc009eb8220)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/pipeline/pipeline.go:75 +0x38

created by github.com/couchbase/indexing/secondary/pipeline.(*Pipeline).runIt

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/pipeline/pipeline.go:74 +0x66

2021-05-30T01:09:27.325-07:00 [Fatal] IndexScanSource - panic detected while processing defnId:14271662457701102502, instId:8283286452070396845, index:bucket1/idx2_SW2Oc, type:scan, partitions:[0], scans: <ud>([{  3 filterRange [{[{  3} {nil nil 0} {  3}]   3 }] <nil>}])</ud>, limit:9223372036854775807, consistency:session_consistency, requestId:e67f0360-ed08-4662-9cc0-6c905a9d1cc4

2021-05-30T01:09:27.345-07:00 [Fatal] goroutine 131704007 [running]:

github.com/couchbase/indexing/secondary/logging.(*destination).StackTraceAll(...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/logging/logging.go:175

github.com/couchbase/indexing/secondary/logging.StackTraceAll(0x215a4e0, 0x10e0001)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/logging/logging.go:317 +0x6d

github.com/couchbase/indexing/secondary/indexer.(*IndexScanSource).Routine.func1(0xc01605b570)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/scan_pipeline.go:131 +0xc5

panic(0x10e7000, 0x20e2790)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/runtime/panic.go:679 +0x1b2

github.com/couchbase/indexing/secondary/indexer.(*plasmaSnapshot).Iterate.func1(0x14f3680, 0xc01ae239e0, 0x14f3680, 0xc01ae23a20, 0xc00a6ab450)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:2780 +0x43c

panic(0x10e7000, 0x20e2790)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/runtime/panic.go:679 +0x1b2

github.com/couchbase/nitro/skiplist.(*Skiplist).GetAccesBarrier(...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/nitro/skiplist/skiplist.go:126

github.com/couchbase/plasma.(*wCtx).EndSMR(...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/smr.go:107

github.com/couchbase/plasma.(*wCtx).EndTx(0xc00e1facf0, 0xc02588dfc0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/smr.go:89 +0x45

github.com/couchbase/plasma.(*MVCCIterator).Close(0xc0238a89c0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/mvcc.go:258 +0x7b

github.com/couchbase/indexing/secondary/indexer.(*plasmaSnapshot).Iterate(0xc00a6ab450, 0x14ec0c0, 0xc01fe3d8a0, 0x14f3680, 0xc01ae239e0, 0x14f3680, 0xc01ae23a20, 0x3, 0x1327118, 0xc00e8f2450, ...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:2849 +0x4f7

github.com/couchbase/indexing/secondary/indexer.(*plasmaSnapshot).Range(0xc00a6ab450, 0x14ec0c0, 0xc01fe3d8a0, 0x14f3680, 0xc01ae239e0, 0x14f3680, 0xc01ae23a20, 0x3, 0xc00e8f2450, 0x10f0ec0, ...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:2763 +0xb8

github.com/couchbase/indexing/secondary/indexer.scanSingleSlice(0xc008a3e680, 0x14f3680, 0xc01ae239e0, 0x14f3680, 0xc01ae23a20, 0x3, 0x12b4051, 0xb, 0xc01f829590, 0x1, ...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/scan_scatter.go:205 +0x233

github.com/couchbase/indexing/secondary/indexer.scanOne(0xc008a3e680, 0x14f3680, 0xc01ae239e0, 0x14f3680, 0xc01ae23a20, 0x3, 0x12b4051, 0xb, 0xc01f829590, 0x1, ...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/scan_scatter.go:147 +0x111

github.com/couchbase/indexing/secondary/indexer.scatter(0xc008a3e680, 0x14f3680, 0xc01ae239e0, 0x14f3680, 0xc01ae23a20, 0x3, 0x12b4051, 0xb, 0xc01f829590, 0x1, ...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/scan_scatter.go:54 +0xc4

github.com/couchbase/indexing/secondary/indexer.(*IndexScanSource).Routine(0xc01605b570, 0x0, 0x0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/scan_pipeline.go:339 +0xdae

github.com/couchbase/indexing/secondary/pipeline.(*Pipeline).runIt.func1(0xc01ae23a60, 0xc009eb8220)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/pipeline/pipeline.go:75 +0x38

created by github.com/couchbase/indexing/secondary/pipeline.(*Pipeline).runIt

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/pipeline/pipeline.go:74 +0x66