GoXDCR: fault at gomemcached/client.(*Client).StatsMap when replication is paused

ToplogyChangeDetector 2015-06-01T00:26:54.116-07:00 [INFO] validateSourceTopology...

unexpected fault address 0x0

fatal error: fault

[signal 0x7 code=0x80 addr=0x0 pc=0x409707]

goroutine 114 [running]:

runtime.gothrow(0x9b4b10, 0x5)

	/usr/local/go/src/runtime/panic.go:503 +0x8e fp=0xc20a367490 sp=0xc20a367478

runtime.sigpanic()

	/usr/local/go/src/runtime/sigpanic_unix.go:23 +0xff fp=0xc20a3674e0 sp=0xc20a367490

runtime.mapassign1(0x826240, 0xc209098870, 0xc20a367620, 0xc20a367630)

	/usr/local/go/src/runtime/hashmap.go:421 +0x217 fp=0xc20a367580 sp=0xc20a3674e0

github.com/couchbase/gomemcached/client.(*Client).StatsMap(0xc2083afa70, 0x9ea0b0, 0xd, 0xc208053760, 0x0, 0x0)

	/home/couchbase/jenkins/workspace/sherlock-unix/godeps/src/github.com/couchbase/gomemcached/client/mc.go:570 +0x192 fp=0xc20a367668 sp=0xc20a367580

github.com/couchbase/goxdcr/pipeline_svc.getHighSeqNos(0xc208318e20, 0x13, 0xc2082ed000, 0x93, 0x100, 0xc2083afa70, 0xc2083afa70, 0x0, 0x0)

	/home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:227 +0x79 fp=0xc20a3676c0 sp=0xc20a367668

github.com/couchbase/goxdcr/pipeline_svc.calculateTotalChanges(0xc208122570, 0xc208544000, 0x0, 0x0, 0xc2084e6540, 0xc208728000, 0x0, 0x0)

	/home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:1063 +0x1d7 fp=0xc20a3677e0 sp=0xc20a3676c0

github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).calculateChangesLeft(0xc20808a460, 0xdbf4d7, 0xe, 0x0, 0x0)

	/home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:533 +0x6d fp=0xc20a367828 sp=0xc20a3677e0

github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).processCalculatedStats(0xc20808a460, 0x7f77d8025538, 0xc2095b7470, 0xc208fd3040, 0x0, 0x0)

	/home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:440 +0x10b fp=0xc20a367a00 sp=0xc20a367828

github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).processRawStats(0xc20808a460, 0x0, 0x0)

	/home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:421 +0x7c9 fp=0xc20a367c98 sp=0xc20a367a00

github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).updateStatsOnce(0xc20808a460, 0x0, 0x0)

	/home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:290 +0x3ae fp=0xc20a367d98 sp=0xc20a367c98

github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).updateStats(0xc20808a460, 0x0, 0x0)

	/home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:266 +0x37c fp=0xc20a367fc8 sp=0xc20a367d98

runtime.goexit()

	/usr/local/go/src/runtime/asm_amd64.s:2232 +0x1 fp=0xc20a367fd0 sp=0xc20a367fc8

created by github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).Start

	/home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:659 +0x367

goroutine 1 [chan receive, 4449 minutes]:

main.main()

	/home/couchbase/jenkins/workspace/sherlock-unix/goproj/src/github.com/couchbase/goxdcr/main/main.go:142 +0x14c8

:

:

:

:

HttpServer 2015-06-01T00:26:57.715-07:00 [INFO] [xdcr:127.0.0.1:9998] new http server xdcr 127.0.0.1:9998 /

AdminPort 2015-06-01T00:26:57.716-07:00 [INFO] http server started 127.0.0.1:9998 !

HttpServer 2015-06-01T00:26:57.716-07:00 [INFO] [xdcr:127.0.0.1:9998] starting ...

RemoteClusterChangeListener 2015-06-01T00:26:57.725-07:00 [INFO] metakvCallback called on listener RemoteClusterChangeListener with path = /remoteCluster/tlmC5tqc2UZ4Vj4r0CyBzRjHGFmky1JWoz_EwwjNM88=

RemoteClusterService 2015-06-01T00:26:57.725-07:00 [INFO] metakvCallback called on path = /remoteCluster/tlmC5tqc2UZ4Vj4r0CyBzRjHGFmky1JWoz_EwwjNM88=

ReplicationSpecChangeListener 2015-06-01T00:26:57.725-07:00 [INFO] metakvCallback called on listener ReplicationSpecChangeListener with path = /replicationSpec/4f8cef9e84a1256b38de62e9d04f1ec2/saslbucket/saslbucket

ReplicationSpecChangeListener 2015-06-01T00:26:57.725-07:00 [INFO] metakvCallback called on listener ReplicationSpecChangeListener with path = /replicationSpec/4f8cef9e84a1256b38de62e9d04f1ec2/standardbucket/standardbucket

ReplicationSpecService 2015-06-01T00:26:57.725-07:00 [INFO] ReplicationSpecServiceCallback called on path = /replicationSpec/4f8cef9e84a1256b38de62e9d04f1ec2/saslbucket/saslbucket

RemoteClusterChangeListener 2015-06-01T00:26:57.725-07:00 [INFO] remoteClusterChangedCallback called on refId = remoteCluster/tlmC5tqc2UZ4Vj4r0CyBzRjHGFmky1JWoz_EwwjNM88=

ReplicationSpecChangeListener 2015-06-01T00:26:57.725-07:00 [INFO] specChangedCallback called on id = 4f8cef9e84a1256b38de62e9d04f1ec2/saslbucket/saslbucket

ReplicationSpecService 2015-06-01T00:26:57.725-07:00 [INFO] ReplicationSpecServiceCallback called on path = /replicationSpec/4f8cef9e84a1256b38de62e9d04f1ec2/standardbucket/standardbucket

ReplicationSpecChangeListener 2015-06-01T00:26:57.725-07:00 [INFO] metakvCallback called on listener ReplicationSpecChangeListener with path = /replicationSpec/4f8cef9e84a1256b38de62e9d04f1ec2/standardbucket1/standardbucket1

ReplicationSpecService 2015-06-01T00:26:57.725-07:00 [INFO] ReplicationSpecServiceCallback called on path = /replicationSpec/4f8cef9e84a1256b38de62e9d04f1ec2/standardbucket1/standardbucket1

ReplicationSpecChangeListener 2015-06-01T00:26:57.726-07:00 [INFO] specChangedCallback called on id = 4f8cef9e84a1256b38de62e9d04f1ec2/standardbucket/standardbucket

ReplicationSpecChangeListener 2015-06-01T00:26:57.726-07:00 [INFO] specChangedCallback called on id = 4f8cef9e84a1256b38de62e9d04f1ec2/standardbucket1/standardbucket1

PipelineManager 2015-06-01T00:27:03.227-07:00 [INFO] Replication Status = map[4f8cef9e84a1256b38de62e9d04f1ec2/standardbucket/standardbucket:name={4f8cef9e84a1256b38de62e9d04f1ec2/standardbucket/standardbucket}, status={Paused}, errors={[]}, progress={}

 4f8cef9e84a1256b38de62e9d04f1ec2/standardbucket1/standardbucket1:name={4f8cef9e84a1256b38de62e9d04f1ec2/standardbucket1/standardbucket1}, status={Paused}, errors={[]}, progress={}

 4f8cef9e84a1256b38de62e9d04f1ec2/saslbucket/saslbucket:name={4f8cef9e84a1256b38de62e9d04f1ec2/saslbucket/saslbucket}, status={Paused}, errors={[]}, progress={}

]

PipelineManager 2015-06-01T00:27:08.212-07:00 [INFO] Replication Status = map[4f8cef9e84a1256b38de62e9d04f1ec2/saslbucket/saslbucket:name={4f8cef9e84a1256b38de62e9d04f1ec2/saslbucket/saslbucket}, status={Paused}, errors={[]}, progress={}

 4f8cef9e84a1256b38de62e9d04f1ec2/standardbucket/standardbucket:name={4f8cef9e84a1256b38de62e9d04f1ec2/standardbucket/standardbucket}, status={Paused}, errors={[]}, progress={}

 4f8cef9e84a1256b38de62e9d04f1ec2/standardbucket1/standardbucket1:name={4f8cef9e84a1256b38de62e9d04f1ec2/standardbucket1/standardbucket1}, status={Paused}, errors={[]}, progress={}

]

PipelineManager 2015-06-01T00:27:13.273-07:00 [INFO] Replication Status = map[4f8cef9e84a1256b38de62e9d04f1ec2/saslbucket/saslbucket:name={4f8cef9e84a1256b38de62e9d04f1ec2/saslbucket/saslbucket}, status={Paused}, errors={[]}, progress={}

 4f8cef9e84a1256b38de62e9d04f1ec2/standardbucket/standardbucket:name={4f8cef9e84a1256b38de62e9d04f1ec2/standardbucket/standardbucket}, status={Paused}, errors={[]}, progress={}

 4f8cef9e84a1256b38de62e9d04f1ec2/standardbucket1/standardbucket1:name={4f8cef9e84a1256b38de62e9d04f1ec2/standardbucket1/standardbucket1}, status={Paused}, errors={[]}, progress={}

]

PipelineManager 2015-06-01T00:27:18.253-07:00 [INFO] Replication Status = map[4f8cef9e84a1256b38de62e9d04f1ec2/standardbucket1/standardbucket1:name={4f8cef9e84a1256b38de62e9d04f1ec2/standardbucket1/standardbucket1}, status={Paused}, errors={[]}, progress={}

 4f8cef9e84a1256b38de62e9d04f1ec2/saslbucket/saslbucket:name={4f8cef9e84a1256b38de62e9d04f1ec2/saslbucket/saslbucket}, status={Paused}, errors={[]}, progress={}

 4f8cef9e84a1256b38de62e9d04f1ec2/standardbucket/standardbucket:name={4f8cef9e84a1256b38de62e9d04f1ec2/standardbucket/standardbucket}, status={Paused}, errors={[]}, progress={}

]

PipelineManager 2015-06-01T00:27:23.194-07:00 [INFO] Replication Status = map[4f8cef9e84a1256b38de62e9d04f1ec2/standardbucket/standardbucket:name={4f8cef9e84a1256b38de62e9d04f1ec2/standardbucket/standardbucket}, status={Paused}, errors={[]}, progress={}

 4f8cef9e84a1256b38de62e9d04f1ec2/standardbucket1/standardbucket1:name={4f8cef9e84a1256b38de62e9d04f1ec2/standardbucket1/standardbucket1}, status={Paused}, errors={[]}, progress={}

 4f8cef9e84a1256b38de62e9d04f1ec2/saslbucket/saslbucket:name={4f8cef9e84a1256b38de62e9d04f1ec2/saslbucket/saslbucket}, status={Paused}, errors={[]}, progress={}

]