[System Test] Service 'eventing' exited with status 2.

-test tests/integration/neo/test_neo_couchstore_milestone3.yml -scope tests/integration/neo/scope_couchstore.yml

[2021-11-11T17:37:24-08:00, sequoiatools/couchbase-cli:7.1:afde4e] server-add -c 172.23.97.74:8091 --server-add https://172.23.96.48 -u Administrator -p password --server-add-username Administrator --server-add-password password --services eventing

[2021-11-11T17:37:38-08:00, sequoiatools/couchbase-cli:7.1:d09cb8] rebalance -c 172.23.97.74:8091 -u Administrator -p password

→  

Error occurred on container - sequoiatools/couchbase-cli:7.1:[rebalance -c 172.23.97.74:8091 -u Administrator -p password]

docker logs d09cb8

docker start d09cb8

*Unable to display progress bar on this os

JERROR: Rebalance failed. See logs for detailed reason. You can try again.

[2021-11-11T17:38:17-08:00, sequoiatools/cmd:46e8b2] 60

[2021-11-11T17:39:24-08:00, sequoiatools/cmd:2920b5] 300

[2021-11-11T17:45:46-08:00, sequoiatools/couchbase-cli:7.1:ece8dd] rebalance -c 172.23.97.74:8091 --server-remove 172.23.120.58 -u Administrator -p password

→  

Error occurred on container - sequoiatools/couchbase-cli:7.1:[rebalance -c 172.23.97.74:8091 --server-remove 172.23.120.58 -u Administrator -p password]

docker logs ece8dd

docker start ece8dd

*Unable to display progress bar on this os

JERROR: Rebalance failed. See logs for detailed reason. You can try again.

[2021-11-11T17:46:39-08:00, sequoiatools/cmd:3df50a] 60

[2021-11-11T17:47:46-08:00, sequoiatools/cmd:4a0808] 300

[2021-11-11T17:56:12-08:00, sequoiatools/couchbase-cli:7.1:9c2d6e] server-add -c 172.23.97.74:8091 --server-add https://172.23.106.136 -u Administrator -p password --server-add-username Administrator --server-add-password password --services eventing

[2021-11-11T17:56:25-08:00, sequoiatools/couchbase-cli:7.1:589f8b] rebalance -c 172.23.97.74:8091 --server-remove 172.23.120.58 -u Administrator -p password

→  

Error occurred on container - sequoiatools/couchbase-cli:7.1:[rebalance -c 172.23.97.74:8091 --server-remove 172.23.120.58 -u Administrator -p password]

docker logs 589f8b

docker start 589f8b

*Unable to display progress bar on this os

JERROR: Rebalance failed. See logs for detailed reason. You can try again.

2021-11-11T17:38:09.747-08:00, ns_orchestrator:0:critical:message(ns_1@172.23.97.74) - Rebalance exited with reason {service_rebalance_failed,eventing,

                              {agent_died,<32771.9174.2>,

                               {lost_connection,

                                {'ns_1@172.23.123.33',shutdown}}}}.

Rebalance Operation Id = 0fc23f760c3236c0ddaf5dfb90d4e2de

2021-11-11T17:38:09.790-08:00, ns_log:0:info:message(ns_1@172.23.123.33) - Service 'eventing' exited with status 2. Restarting. Messages:

github.com/couchbase/cbauth/service.serviceAPI.GetCurrentTopology(0x1175140, 0xc00047b000, 0x0, 0x0, 0xc007de5820, 0x8, 0x9, 0x7530, 0xc005236500, 0xc000a0dbb0, ...)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/service/revrpc.go:77 +0x103

reflect.Value.call(0xc00033a900, 0xc000128098, 0x13, 0xfc8c9f, 0x4, 0xc000a0df18, 0x3, 0x3, 0xc000692f01, 0xc0063401e0, ...)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/reflect/value.go:460 +0x5f6

reflect.Value.Call(0xc00033a900, 0xc000128098, 0x13, 0xc000692f18, 0x3, 0x3, 0x8, 0x0, 0x0)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/reflect/value.go:321 +0xb4

net/rpc.(*service).call(0xc000136000, 0xc00013e190, 0xc00063a088, 0xc00063a090, 0xc000158100, 0xc00063c380, 0xea5420, 0xc003561800, 0x199, 0xde5580, ...)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/net/rpc/server.go:377 +0x16f

created by net/rpc.(*Server).ServeCodec

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/net/rpc/server.go:474 +0x42b

2021-11-11T17:38:09.636-08:00 [Info] Consumer::checkAndAddToEnqueMap [worker_n1ql_0_0:/tmp/127.0.0.1:8091_0_1121266589.sock:121179] vb: 610 enqueuing

goroutine 625325 [running]:

runtime.throw(0xfe6b76, 0x21)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/runtime/panic.go:774 +0x72 fp=0xc000107048 sp=0xc000107018 pc=0x432f22

runtime.mapaccess2(0xe538a0, 0xc002db7c50, 0xc0001070c6, 0xd442324, 0x2)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/runtime/map.go:470 +0x278 fp=0xc000107090 sp=0xc000107048 pc=0x411968

github.com/couchbase/eventing/consumer.(*Consumer).checkAndAddToEnqueueMap(0xc0018bad00, 0x670219, 0x0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/util.go:38 +0xd1 fp=0xc000107208 sp=0xc000107090 pc=0xc6b601

github.com/couchbase/eventing/consumer.(*Consumer).updateVbOwnerAndStartDCPStream(0xc0018bad00, 0xc006097560, 0xf, 0xc000100219, 0xc002477200, 0x6, 0x0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/vbucket_takeover.go:347 +0x2c1 fp=0xc000107370 sp=0xc000107208 pc=0xc7b881

github.com/couchbase/eventing/consumer.(*Consumer).doVbTakeover(0xc0018bad00, 0xc000100219, 0x1, 0x1)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/vbucket_takeover.go:312 +0x143e fp=0xc000107ad8 sp=0xc000107370 pc=0xc799be

github.com/couchbase/eventing/consumer.glob..func1(0xc005c387e0, 0x2, 0x2, 0xf, 0xc000786fa8)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/bucket_ops.go:26 +0xa2 fp=0xc000107ca8 sp=0xc000107ad8 pc=0xc80582

github.com/couchbase/eventing/util.Retry(0x11651a0, 0xc006097520, 0xc000e47260, 0x1019130, 0xc005c387e0, 0x2, 0x2, 0x0, 0x0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/util/retry.go:65 +0x12a fp=0xc000107d18 sp=0xc000107ca8 pc=0x939bba

github.com/couchbase/eventing/consumer.(*Consumer).vbsStateUpdate.func2(0xfdc483, 0x18, 0xc0018bad00, 0x1, 0xc005224400, 0x3b, 0x40, 0xc005f466d0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/vbucket_takeover.go:164 +0x59b fp=0xc000107fa0 sp=0xc000107d18 pc=0xca0bcb

runtime.goexit()

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/runtime/asm_amd64.s:1357 +0x1 fp=0xc000107fa8 sp=0xc000107fa0 pc=0x462cc1

created by github.com/couchbase/eventing/consumer.(*Consumer).vbsStateUpdate

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/vbucket_takeover.go:127 +0xb6d

2021-11-11T17:46:31.409-08:00, ns_orchestrator:0:critical:message(ns_1@172.23.97.74) - Rebalance exited with reason {service_rebalance_failed,eventing,

                              {agent_died,<32758.27284.79>,

                               {lost_connection,

                                {'ns_1@172.23.96.48',shutdown}}}}.

Rebalance Operation Id = 1393a7d25dbe5a5f83c61a7b0eae750e

2021-11-11T17:46:31.530-08:00, ns_log:0:info:message(ns_1@172.23.96.48) - Service 'eventing' exited with status 2. Restarting. Messages:

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/service/revrpc.go:38 +0xef

goroutine 25808 [sleep]:

runtime.goparkunlock(...)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/runtime/proc.go:310

time.Sleep(0x3b9aca00)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/runtime/time.go:105 +0x157

github.com/couchbase/eventing/util.Retry(0x11651a0, 0xc0061aeb48, 0xc0021b67e0, 0x1019130, 0xc000a8dea0, 0x2, 0x2, 0x0, 0x0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/util/retry.go:81 +0xe9

github.com/couchbase/eventing/consumer.(*Consumer).vbsStateUpdate.func2(0xfdc483, 0x18, 0xc001dd4d00, 0x1, 0xc000760fc0, 0x1d, 0x20, 0xc003221ff0)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/vbucket_takeover.go:164 +0x59b

created by github.com/couchbase/eventing/consumer.(*Consumer).vbsStateUpdate

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/vbucket_takeover.go:127 +0xb6d

2021-11-11T17:46:31.339-08:00 [Info] Consumer::checkIfAlreadyEnqueued [worker_timers_0_0:/tmp/127.0.0.1:8091_0_1613531711.sock:384] vb: 725 enqueuing

fatal error: concurrent map read and map write

goroutine 25813 [running]:

runtime.throw(0xfe6b76, 0x21)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/runtime/panic.go:774 +0x72 fp=0xc0028bbb58 sp=0xc0028bbb28 pc=0x432f22

runtime.mapaccess2(0xe538a0, 0xc002da18c0, 0xc0028bbbd6, 0xc0028bbc68, 0x5)

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/runtime/map.go:470 +0x278 fp=0xc0028bbba0 sp=0xc0028bbb58 pc=0x411968

github.com/couchbase/eventing/consumer.(*Consumer).checkIfAlreadyEnqueued(0xc0037e3600, 0xc002da02c8, 0xc0028bbd00)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/util.go:20 +0xd1 fp=0xc0028bbd18 sp=0xc0028bbba0 pc=0xc6b021

github.com/couchbase/eventing/consumer.(*Consumer).vbsStateUpdate.func2(0xfdc483, 0x18, 0xc0037e3600, 0x2, 0xc000040800, 0x2e, 0x40, 0xc006471310)

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/vbucket_takeover.go:156 +0x46e fp=0xc0028bbfa0 sp=0xc0028bbd18 pc=0xca0a9e

runtime.goexit()

	/home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/runtime/asm_amd64.s:1357 +0x1 fp=0xc0028bbfa8 sp=0xc0028bbfa0 pc=0x462cc1

created by github.com/couchbase/eventing/consumer.(*Consumer).vbsStateUpdate

	/home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/vbucket_takeover.go:127 +0xb6d