[System Test] : fatal error: concurrent map iteration and map write

./sequoia -client 172.23.97.180:2375 -provider file:centos_second_cluster.yml -test tests/integration/7.2/test_7.2.yml -scope tests/integration/7.2/scope_7.2_magma.yml -scale 1 -repeat 0 -log_level 0 -version 7.2.6-8104 -skip_setup=false -skip_test=false -skip_teardown=true -skip_cleanup=false -continue=false -collect_on_error=false -stop_on_error=false -duration=604800 -show_topology=true

Service 'n1ql' exited with status 2. Restarting. Messages:

github.com/couchbase/query/datastore/couchbase.(*namespace).KeyspaceByName(0xc000e06400?, {0xc000eff290?, 0x8?})

/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/datastore/couchbase/couchbase.go:917 +0x18

github.com/couchbase/query/datastore/system.(*keyspaceIndex).ScanEntries(0xc000ed9a70, {0x275c601?, 0xc00469a601?}, 0x7fffffffffffffff, {0x1405d25?, 0xc0040ecea0?}, {0xc01150cdb0?, 0xc00469a720?}, 0xc009c5e7e0)

/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/datastore/system/system_keyspace_keyspaces.go:475 +0x245

github.com/couchbase/query/execution.(*PrimaryScan).scanEntries(0xc0165c0848, 0xc0165c0008, 0xc009c5e7e0, 0x7fffffffffffffff)

/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/scan_primary.go:202 +0x12d

created by github.com/couchbase/query/execution.(*PrimaryScan).scanPrimary in goroutine 177498018

/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/scan_primary.go:93 +0x1d4

fatal error: concurrent map iteration and map write

goroutine 177489282 [running]:

github.com/couchbase/query/datastore/couchbase.refreshScopesAndCollections(0xc000ac8280, 0xc001181e00)

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/datastore/couchbase/collections.go:533 +0xf5a

github.com/couchbase/query/datastore/couchbase.(*namespace).keyspaceByName(0xc00026c0e0, {0xc000eff290, 0x7})

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/datastore/couchbase/couchbase.go:948 +0x53e

github.com/couchbase/query/datastore/couchbase.(*namespace).KeyspaceByName(0xc0127b4d80?, {0xc000eff290?, 0x4?})

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/datastore/couchbase/couchbase.go:917 +0x18

github.com/couchbase/query/datastore/system.(*keyspaceIndex).ScanEntries(0xc000ed9a70, {0x275c601?, 0xc002d06f01?}, 0x7fffffffffffffff, {0x1405d25?, 0xc014100ea0?}, {0xc012dbf8d0?, 0xc002d06fb8?}, 0xc0127b4d80)

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/datastore/system/system_keyspace_keyspaces.go:475 +0x245

github.com/couchbase/query/execution.(*PrimaryScan).scanEntries(0xc001417348, 0xc001416b08, 0xc0127b4d80, 0x7fffffffffffffff)

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/scan_primary.go:202 +0x12d

created by github.com/couchbase/query/execution.(*PrimaryScan).scanPrimary in goroutine 177489249

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/scan_primary.go:93 +0x1d4

goroutine 1 [chan receive, 2760 minutes]:

main.signalCatcher(0xc000f66b48, 0x20?)

        goproj/src/github.com/couchbase/query/server/cbq-engine/main.go:437 +0x9e

main.main()

        goproj/src/github.com/couchbase/query/server/cbq-engine/main.go:427 +0x15fa

goroutine 9 [select, 2760 minutes]:

github.com/blevesearch/bleve_index_api.AnalysisWorker(...)

        /home/couchbase/.cbdepscache/gomodcache/pkg/mod/github.com/blevesearch/bleve_index_api@v1.0.3-0.20240624205006-07f7b7930fd5/analysis.go:46

created by github.com/blevesearch/bleve_index_api.NewAnalysisQueue in goroutine 1

        /home/couchbase/.cbdepscache/gomodcache/pkg/mod/github.com/blevesearch/bleve_index_api@v1.0.3-0.20240624205006-07f7b7930fd5/analysis.go:38 +0xa5

goroutine 10 [select, 2760 minutes]: 

github.com/blevesearch/bleve_index_api.AnalysisWorker(...)

        /home/couchbase/.cbdepscache/gomodcache/pkg/mod/github.com/blevesearch/bleve_index_api@v1.0.3-0.20240624205006-07f7b7930fd5/analysis.go:46

created by github.com/blevesearch/bleve_index_api.NewAnalysisQueue in goroutine 1

        /home/couchbase/.cbdepscache/gomodcache/pkg/mod/github.com/blevesearch/bleve_index_api@v1.0.3-0.20240624205006-07f7b7930fd5/analysis.go:38 +0xa5

goroutine 11 [select, 2760 minutes]: 

github.com/blevesearch/bleve_index_api.AnalysisWorker(...)

        /home/couchbase/.cbdepscache/gomodcache/pkg/mod/github.com/blevesearch/bleve_index_api@v1.0.3-0.20240624205006-07f7b7930fd5/analysis.go:46

created by github.com/blevesearch/bleve_index_api.NewAnalysisQueue in goroutine 1

        /home/couchbase/.cbdepscache/gomodcache/pkg/mod/github.com/blevesearch/bleve_index_api@v1.0.3-0.20240624205006-07f7b7930fd5/analysis.go:38 +0xa5

goroutine 12 [select, 2760 minutes]:

github.com/blevesearch/bleve_index_api.AnalysisWorker(...)

        /home/couchbase/.cbdepscache/gomodcache/pkg/mod/github.com/blevesearch/bleve_index_api@v1.0.3-0.20240624205006-07f7b7930fd5/analysis.go:46

created by github.com/blevesearch/bleve_index_api.NewAnalysisQueue in goroutine 1

        /home/couchbase/.cbdepscache/gomodcache/pkg/mod/github.com/blevesearch/bleve_index_api@v1.0.3-0.20240624205006-07f7b7930fd5/analysis.go:38 +0xa5

goroutine 13 [select, 2760 minutes]:

github.com/couchbase/cbauth/cbauthimpl.(*tlsNotifier).loop(0xc00060f2f0)

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/cbauthimpl/impl.go:410 +0x65

created by github.com/couchbase/cbauth/cbauthimpl.NewSVCForTest in goroutine 1

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/cbauthimpl/impl.go:637 +0x365

goroutine 14 [select, 2760 minutes]:

github.com/couchbase/cbauth/cbauthimpl.(*cfgChangeNotifier).loop(0xc00060f308)

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/cbauthimpl/impl.go:329 +0x7c

created by github.com/couchbase/cbauth/cbauthimpl.NewSVCForTest in goroutine 1

        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/cbauthimpl/impl.go:638 +0x3b6