System Test : Panic in indexer when rebalancing out 2 indexer nodes together

2018-03-26T02:04:07.300-07:00 [Info] Rebalancer::decodeTransferToken TransferToken TransferToken6d:23:d0:73:94:12:73:ff  MasterId: 7e151aa742a2b2240892f9746ed18211 SourceId: ce32eb3a41c3d0f1e646223ad15b0e1d DestId: bb3cb0f098458e884e36e2fd195b44b0 RebalId: 4244b992a7487fb8923cb3fe821042b7 State: TransferTokenDeleted BuildSource: Dcp TransferMode: Move InstId: 4777481747353977909 RealInstId: 0 Partitions: [0]Inst:

        InstId: 4777481747353977909

        Defn: DefnId: 8163621099742918705 Name: o2_claims Using: plasma Bucket: other-2 IsPrimary: false NumReplica: 1 InstVersion: 1

                SecExprs: <ud>([`claim`])</ud>

                Desc: [false]

                PartitionScheme: SINGLE

                HashScheme: CRC32 PartitionKeys: [] WhereExpr: <ud>()</ud> RetainDeletedXATTR: false

        State: INDEX_STATE_ACTIVE

        RState: RebalActive

        Stream: NIL_STREAM

        Version: 0

        ReplicaId: 0

        PartitionContainer: <nil>

2018-03-26T02:04:07.508-07:00 [Info] Rebalancer::processTokens Received empty or deleted transfer token /indexing/rebalance/TransferToken6d:23:d0:73:94:12:73:ff

panic: runtime error: invalid memory address or nil pointer dereference

[signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x969912]

goroutine 6095793 [running]:

panic(0xdf81c0, 0xc4200160c0)

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/panic.go:500 +0x1a1 fp=0xc49b987750 sp=0xc49b9876c0

runtime.panicmem()

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/panic.go:62 +0x6d fp=0xc49b987780 sp=0xc49b987750

runtime.sigpanic()

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/sigpanic_unix.go:24 +0x214 fp=0xc49b9877d8 sp=0xc49b987780

github.com/couchbase/plasma.(*Snapshot).Close(0x0)

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/mvcc.go:173 +0x22 fp=0xc49b987800 sp=0xc49b9877d8

github.com/couchbase/plasma.(*Snapshot).Close(0xc43b155540)

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/mvcc.go:175 +0x77 fp=0xc49b987828 sp=0xc49b987800

github.com/couchbase/plasma.(*Snapshot).Close(0xc44fbe99a0)

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/mvcc.go:175 +0x77 fp=0xc49b987850 sp=0xc49b987828

github.com/couchbase/plasma.(*MVCCIterator).Close(0xc4ac0abce0)

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/plasma/mvcc.go:219 +0x2e fp=0xc49b987870 sp=0xc49b987850

github.com/couchbase/indexing/secondary/indexer.(*plasmaSnapshot).Iterate(0xc43b155590, 0x17ac400, 0xc47d000b00, 0x17b04e0, 0xc47d000b80, 0x17b04e0, 0xc47d000bc0, 0x3, 0x100db18, 0xc45007e540, ...)

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:1773 +0x3d4 fp=0xc49b9878f0 sp=0xc49b987870

github.com/couchbase/indexing/secondary/indexer.(*plasmaSnapshot).Range(0xc43b155590, 0x17ac400, 0xc47d000b00, 0x17b04e0, 0xc47d000b80, 0x17b04e0, 0xc47d000bc0, 0x3, 0xc45007e540, 0xc4c26c8000, ...)

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/plasma_slice.go:1699 +0xb2 fp=0xc49b987960 sp=0xc49b9878f0

github.com/couchbase/indexing/secondary/indexer.scanSingleSlice(0xc4b1bdad80, 0x17b04e0, 0xc47d000b80, 0x17b04e0, 0xc47d000bc0, 0x3, 0xf359d4, 0x5, 0xc425b0e050, 0x1, ...)

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/scan_scatter.go:164 +0x41a fp=0xc49b987a20 sp=0xc49b987960

github.com/couchbase/indexing/secondary/indexer.scanOne(0xc4b1bdad80, 0x17b04e0, 0xc47d000b80, 0x17b04e0, 0xc47d000bc0, 0x3, 0xf359d4, 0x5, 0xc425b0e050, 0x1, ...)

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/scan_scatter.go:110 +0x10e fp=0xc49b987b30 sp=0xc49b987a20

github.com/couchbase/indexing/secondary/indexer.scatter(0xc4b1bdad80, 0x17b04e0, 0xc47d000b80, 0x17b04e0, 0xc47d000bc0, 0x3, 0xf359d4, 0x5, 0xc425b0e050, 0x1, ...)

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/scan_scatter.go:40 +0xaa fp=0xc49b987be0 sp=0xc49b987b30

github.com/couchbase/indexing/secondary/indexer.(*IndexScanSource).Routine(0xc43fc803f0, 0x0, 0x0)

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/indexer/scan_pipeline.go:293 +0x93d fp=0xc49b987f28 sp=0xc49b987be0

github.com/couchbase/indexing/secondary/pipeline.(*Pipeline).runIt.func1(0xc4b569be80, 0xc4809f0640)

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/pipeline/pipeline.go:75 +0x45 fp=0xc49b987f90 sp=0xc49b987f28

runtime.goexit()

        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.7.6/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc49b987f98 sp=0xc49b987f90

created by github.com/couchbase/indexing/secondary/pipeline.(*Pipeline).runIt

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/indexing/secondary/pipeline/pipeline.go:80 +0x66