[System Test] fatal error: concurrent map iteration and map write

Build - 7.1.0-1085
Test -

-test tests/integration/neo/test_neo_kv_gsi_coll_xdcr_backup_sgw _fts_itemct_txns_eventing _cbas_scale3_magma.yml -scope tests/integration/neo/scope_neo_with_backup_magma.yml

Scale - 3
Day - 1

On 172.23.120.73

• goxdcr.log

  2021-07-23T04:32:13.970-07:00 INFO GOXDCR.PipelineMgr: checking pipeline spec=Id: 408030b35137f655c784fa33ad2b65ef/bucket8/bucket8 InternalId: UipNTermtE7ikUFcGN5-fg== SourceBucketName: bucket8 SourceBucketUUID: c6e404a741e471445abf0be01fa82259 TargetClusterUUID: 408030b35137f655c784fa33ad2b65ef TargetBucketName: bucket8 TargetBucketUUID: ba8ee5356e6a54fce1733700026ee014 Settings: map[CollectionsMgtMulti:ExplicitMapping: false Mirroring: false Migration: false OSO: true active:true backlogThreshold:50 bandwidth_limit:0 checkpoint_interval:600 colMappingRules:map[] collectionsExplicitMapping:false collectionsMigrationMode:false collectionsMirroringMode:false collectionsOSOMode:true collectionsSkipSrcValidation:false compression_type:3 delAllBackfills:false delSpecificBackfillForVb:-1 dismissEvent:-1 doc_batch_size_kb:2048 failure_restart_interval:10 filterBypassExpiry:false filterDeletion:false filterExpiration:false filter_exp_del:0 filter_expression: filter_expression_version:0 filter_skip_restream:false hlvPruningWindowSec:259200 jsFunctionTimeoutMs:20000 log_level:Info manualBackfill: mergeFunctionMapping:map[] optimistic_replication_threshold:256 priority:High replication_type:xmem retryOnRemoteAuthErr:true retryOnRemoteAuthErrMaxWaitSec:3600 source_nozzle_per_node:2 stats_interval:1000 target_nozzle_per_node:2 worker_batch_size:500], source bucket uuid=c6e404a741e471445abf0be01fa82259

  fatal error: concurrent map iteration and map write

  goroutine 85 [running]:

  runtime.throw(0xc9dd3f, 0x26)

  /home/couchbase/.cbdepscache/exploded/x86_64/go-1.15.8/go/src/runtime/panic.go:1116 +0x72 fp=0xc000a63168 sp=0xc000a63138 pc=0x439e32

  runtime.mapiternext(0xc000a63238)

  /home/couchbase/.cbdepscache/exploded/x86_64/go-1.15.8/go/src/runtime/map.go:853 +0x554 fp=0xc000a631e8 sp=0xc000a63168 pc=0x413554

  github.com/couchbase/goxdcr/metadata.(*Settings).Clone(0xc00085a630, 0xba7920)

  /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata/settings_common.go:163 +0xe5 fp=0xc000a632a8 sp=0xc000a631e8 pc=0x839d65

  github.com/couchbase/goxdcr/metadata.(*ReplicationSettings).Clone(0xc000184160, 0xba7920)

  /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata/replication_settings.go:514 +0x2e fp=0xc000a632d8 sp=0xc000a632a8 pc=0x8308ce

  github.com/couchbase/goxdcr/metadata.(*ReplicationSpecification).Clone(0xc0002327e0, 0xc000e92511)

  /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata/replication_spec.go:120 +0x3c fp=0xc000a63300 sp=0xc000a632d8 pc=0x83861c

  github.com/couchbase/goxdcr/metadata_svc.(*ReplicationSpecService).ReplicationSpec(0xc00033c000, 0xc000e92511, 0x30, 0x203000, 0x0, 0x100000000000000)

  /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/replication_spec_service.go:206 +0x7f fp=0xc000a63340 sp=0xc000a63300 pc=0x9aa7ff

  github.com/couchbase/goxdcr/service_def.ReplicationSpecSvc.ReplicationSpec-fm(0xc000e92511, 0x30, 0x48ae65, 0x2, 0x1374240)

  /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_def/replication_spec_svc.go:17 +0x43 fp=0xc000a63380 sp=0xc000a63340 pc=0xa17da3

  github.com/couchbase/goxdcr/pipeline.(*ReplicationStatus).Spec(0xc0018b8120, 0xc000446420)

  /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/ReplicationStatus.go:226 +0x5a fp=0xc000a63420 sp=0xc000a63380 pc=0x9eff1a

  github.com/couchbase/goxdcr/pipeline.(*ReplicationStatus).RuntimeStatus(0xc0018b8120, 0x0, 0x0)

  /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/ReplicationStatus.go:314 +0x97 fp=0xc000a63468 sp=0xc000a63420 pc=0x9f0c77

  github.com/couchbase/goxdcr/pipeline.(*ReplicationStatus).String(0xc0018b8120, 0x0, 0x0)

  /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/ReplicationStatus.go:573 +0x9e fp=0xc000a63550 sp=0xc000a63468 pc=0x9f2efe

  fmt.(*pp).handleMethods(0xc001393ad0, 0xc000000076, 0x1)

  /home/couchbase/.cbdepscache/exploded/x86_64/go-1.15.8/go/src/fmt/print.go:630 +0x30a fp=0xc000a637c0 sp=0xc000a63550 pc=0x4e776a

  fmt.(*pp).printValue(0xc001393ad0, 0xc6a680, 0xc0018b8120, 0x16, 0x76, 0x1)

  /home/couchbase/.cbdepscache/exploded/x86_64/go-1.15.8/go/src/fmt/print.go:727 +0x25a5 fp=0xc000a63998 sp=0xc000a637c0 pc=0x4eaa45

  fmt.(*pp).printValue(0xc001393ad0, 0xba64e0, 0xc0008f40f0, 0x15, 0xffffffff00000076, 0x0)

  /home/couchbase/.cbdepscache/exploded/x86_64/go-1.15.8/go/src/fmt/print.go:784 +0xe38 fp=0xc000a63b70 sp=0xc000a63998 pc=0x4e92d8

  fmt.(*pp).printArg(0xc001393ad0, 0xba64e0, 0xc0008f40f0, 0xc000000076)

  /home/couchbase/.cbdepscache/exploded/x86_64/go-1.15.8/go/src/fmt/print.go:716 +0x2b4 fp=0xc000a63c08 sp=0xc000a63b70 pc=0x4e7df4

  fmt.(*pp).doPrintf(0xc001393ad0, 0xc00237f720, 0x4f, 0xc000a63e50, 0x1, 0x1)

  /home/couchbase/.cbdepscache/exploded/x86_64/go-1.15.8/go/src/fmt/print.go:1030 +0x168 fp=0xc000a63cf0 sp=0xc000a63c08 pc=0x4eb3e8

  fmt.Sprintf(0xc00237f720, 0x4f, 0xc000a63e50, 0x1, 0x1, 0xc00237f720, 0x4f)

  /home/couchbase/.cbdepscache/exploded/x86_64/go-1.15.8/go/src/fmt/print.go:219 +0x66 fp=0xc000a63d48 sp=0xc000a63cf0 pc=0x4e4686

  log.(*Logger).Printf(0xc00025dea0, 0xc00237f720, 0x4f, 0xc000a63e50, 0x1, 0x1)

  /home/couchbase/.cbdepscache/exploded/x86_64/go-1.15.8/go/src/log/log.go:188 +0x53 fp=0xc000a63d90 sp=0xc000a63d48 pc=0x681633

  github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf(0xc000207960, 0xd, 0xc8f1c9, 0x18, 0xc000a63e50, 0x1, 0x1)

  /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 +0x12e fp=0xc000a63e10 sp=0xc000a63d90 pc=0x756aee

  github.com/couchbase/goxdcr/log.(*CommonLogger).Infof(...)

  /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:193

  github.com/couchbase/goxdcr/pipeline_manager.LogStatusSummary()

  /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:275 +0xa5 fp=0xc000a63e70 sp=0xc000a63e10 pc=0xa06ba5

  github.com/couchbase/goxdcr/pipeline_manager.(*PipelineManager).CheckPipelines(0xc00023ac30)

  /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:331 +0x4c fp=0xc000a63e88 sp=0xc000a63e70 pc=0xa072ec

  github.com/couchbase/goxdcr/replication_manager.(*replicationManager).checkReplicationStatus(0x1373fa0, 0xc000258620)

  /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/replication_manager/replication_manager.go:423 +0x2ca fp=0xc000a63fd0 sp=0xc000a63e88 pc=0xaf5fea

  runtime.goexit()

• diag.log

  2021-07-23T04:29:42.425-07:00, xdcr:0:info:message(ns_1@172.23.97.74) - Replication from bucket "default" to bucket "remote" on cluster "remote" created.

  2021-07-23T04:29:48.469-07:00, xdcr:0:info:message(ns_1@172.23.97.74) - Replication from bucket "bucket4" to bucket "bucket4" on cluster "remote" created.

  2021-07-23T04:29:54.506-07:00, xdcr:0:info:message(ns_1@172.23.97.74) - Replication from bucket "bucket8" to bucket "bucket8" on cluster "remote" created.

  2021-07-23T04:30:00.527-07:00, xdcr:0:info:message(ns_1@172.23.97.74) - Replication from bucket "bucket9" to bucket "bucket9" on cluster "remote" created.

  2021-07-23T04:31:14.772-07:00, ns_log:0:info:message(ns_1@172.23.97.148) - Service 'goxdcr' exited with status 2. Restarting. Messages:

  net/http.(*persistConn).Read(0xc0000aa6c0, 0xc000690000, 0x1000, 0x1000, 0xc0002b0780, 0xc0003acc58, 0x407ff5)

  /home/couchbase/.cbdepscache/exploded/x86_64/go-1.15.8/go/src/net/http/transport.go:1887 +0x77

  bufio.(*Reader).fill(0xc00042e360)

  /home/couchbase/.cbdepscache/exploded/x86_64/go-1.15.8/go/src/bufio/bufio.go:101 +0x105

  bufio.(*Reader).Peek(0xc00042e360, 0x1, 0x0, 0x0, 0x1, 0x0, 0xc00021a480)

  /home/couchbase/.cbdepscache/exploded/x86_64/go-1.15.8/go/src/bufio/bufio.go:139 +0x4f

  net/http.(*persistConn).readLoop(0xc0000aa6c0)

  /home/couchbase/.cbdepscache/exploded/x86_64/go-1.15.8/go/src/net/http/transport.go:2040 +0x1a8

  created by net/http.(*Transport).dialConn

  /home/couchbase/.cbdepscache/exploded/x86_64/go-1.15.8/go/src/net/http/transport.go:1708 +0xcb7