goroutine profile: total 4229 984 @ 0x43e876 0x44ee3e 0x90dbf3 0x4711c1 # 0x90dbf2 github.com/couchbase/goxdcr/component.(*AsyncComponentEventListenerImpl).run+0x92 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/component/async_listener.go:68 242 @ 0x43e876 0x44ee3e 0x96b19d 0x4711c1 # 0x96b19c github.com/couchbase/goxdcr/metadata_svc.(*CheckpointsServiceCacheImpl).Run+0x17c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/checkpointsServiceCache.go:263 166 @ 0x43e876 0x44ee3e 0x7282b2 0x4711c1 # 0x7282b1 net/http.(*persistConn).writeLoop+0xf1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:2410 159 @ 0x43e876 0x437117 0x46b6c9 0x4dc712 0x4ddaf9 0x4ddae7 0x5d6b09 0x5e6c65 0x7257ce 0x69251f 0x69267d 0x7265ec 0x4711c1 # 0x46b6c8 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dc711 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4ddaf8 internal/poll.(*pollDesc).waitRead+0x298 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4ddae6 internal/poll.(*FD).Read+0x286 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:167 # 0x5d6b08 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e6c64 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x7257cd net/http.(*persistConn).Read+0x4d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:1943 # 0x69251e bufio.(*Reader).fill+0xfe /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/bufio/bufio.go:106 # 0x69267c bufio.(*Reader).Peek+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/bufio/bufio.go:144 # 0x7265eb net/http.(*persistConn).readLoop+0x1ab /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:2107 121 @ 0x43e876 0x44ee3e 0x729154 0x71d07a 0x705219 0x6c5d37 0x6c55bb 0x6c78bb 0x8d87d3 0x8d87cb 0x8d943e 0xda80c9 0xda68e5 0x9eacbd 0x9e9c4c 0x4711c1 # 0x729153 net/http.(*persistConn).roundTrip+0x993 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:2638 # 0x71d079 net/http.(*Transport).roundTrip+0x7f9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:603 # 0x705218 net/http.(*Transport).RoundTrip+0x18 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/roundtrip.go:17 # 0x6c5d36 net/http.send+0x5f6 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:252 # 0x6c55ba net/http.(*Client).send+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:176 # 0x6c78ba net/http.(*Client).do+0x8fa /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:716 # 0x8d87d2 net/http.(*Client).Do+0x52 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:582 # 0x8d87ca github.com/couchbase/goxdcr/utils.(*Utilities).doRestCall+0x4a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2234 # 0x8d943d github.com/couchbase/goxdcr/utils.(*Utilities).InvokeRestWithRetryWithAuth+0x37d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2321 # 0xda80c8 github.com/couchbase/goxdcr/service_impl.(*CAPIService).send_post+0x308 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/capi_service.go:291 # 0xda68e4 github.com/couchbase/goxdcr/service_impl.(*CAPIService).PreReplicate+0x144 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/capi_service.go:122 # 0x9eacbc github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).getDataFromCkpts+0x4bc /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1442 # 0x9e9c4b github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).startSeqnoGetter+0x2cb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1319 121 @ 0x43e876 0x44ee3e 0x8fd13d 0x4711c1 # 0x8fd13c github.com/couchbase/goxdcr/peerToPeer.(*ReplicaCacheMonitor).Run+0x9c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/replicaCache.go:108 121 @ 0x43e876 0x44ee3e 0x9004f8 0x4711c1 # 0x9004f7 github.com/couchbase/goxdcr/peerToPeer.(*ReplicatorAgentImpl).run+0x457 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/replicaReplicator.go:508 121 @ 0x43e876 0x44ee3e 0x9782de 0x4711c1 # 0x9782dd github.com/couchbase/goxdcr/metadata_svc.(*CollectionsManifestAgent).runPeriodicRefresh+0xdd /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/collections_manifest_service.go:504 121 @ 0x43e876 0x44ee3e 0x97845f 0x4711c1 # 0x97845e github.com/couchbase/goxdcr/metadata_svc.(*CollectionsManifestAgent).runPersistRequestHandler+0xbe /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/collections_manifest_service.go:528 121 @ 0x43e876 0x44ee3e 0xa55845 0x4711c1 # 0xa55844 github.com/couchbase/goxdcr/backfill_manager.(*BackfillRequestHandler).run+0x2a4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/backfill_manager/backfill_request_handler.go:241 102 @ 0x43e876 0x437117 0x46b6c9 0x4dc712 0x4ddaf9 0x4ddae7 0x5d6b09 0x5e6c65 0x4ba65a 0x68b539 0x68b503 0x744545 0x74450d 0x73c4b3 0x95286b 0x94bd85 0x4711c1 # 0x46b6c8 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dc711 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4ddaf8 internal/poll.(*pollDesc).waitRead+0x298 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4ddae6 internal/poll.(*FD).Read+0x286 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:167 # 0x5d6b08 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e6c64 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x4ba659 io.ReadAtLeast+0x99 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:332 # 0x68b538 io.ReadFull+0xd8 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:351 # 0x68b502 github.com/couchbase/gomemcached.(*MCResponse).ReceiveWithBuf+0xa2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:169 # 0x744544 github.com/couchbase/gomemcached.(*MCResponse).Receive+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:156 # 0x74450c github.com/couchbase/gomemcached/client.getResponse+0x6c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/transport.go:35 # 0x73c4b2 github.com/couchbase/gomemcached/client.(*Client).Receive+0x52 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:333 # 0x95286a github.com/couchbase/goxdcr/parts.(*XmemNozzle).readFromClient+0x18a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3498 # 0x94bd84 github.com/couchbase/goxdcr/parts.(*XmemNozzle).receiveResponse+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2618 96 @ 0x43e876 0x44ee3e 0x746959 0x4711c1 # 0x746958 github.com/couchbase/gomemcached/client.(*UprFeed).sendCommands+0xb8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/upr_feed.go:346 85 @ 0x43e876 0x437117 0x46b6c9 0x4dc712 0x4dfd36 0x4dfd23 0x5d7389 0x5e6e45 0x952482 0x951a49 0x9496ec 0x95000b 0x939314 0x94fa89 0x4711c1 # 0x46b6c8 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dc711 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4dfd35 internal/poll.(*pollDesc).waitWrite+0x2f5 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:93 # 0x4dfd22 internal/poll.(*FD).Write+0x2e2 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:391 # 0x5d7388 net.(*netFD).Write+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:96 # 0x5e6e44 net.(*conn).Write+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:195 # 0x952481 github.com/couchbase/goxdcr/parts.(*XmemNozzle).writeToClientWithoutThrottling+0xc1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3463 # 0x951a48 github.com/couchbase/goxdcr/parts.(*XmemNozzle).writeToClient+0x388 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3340 # 0x9496eb github.com/couchbase/goxdcr/parts.(*XmemNozzle).sendSingleSetMeta+0xab /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2334 # 0x95000a github.com/couchbase/goxdcr/parts.(*XmemNozzle).resendIfTimeout+0x46a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3051 # 0x939313 github.com/couchbase/goxdcr/parts.(*requestBuffer).modSlot+0x53 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:296 # 0x94fa88 github.com/couchbase/goxdcr/parts.(*XmemNozzle).checkAndRepairBufferMonitor+0x348 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2995 72 @ 0x43e876 0x437117 0x46b6c9 0x4dc712 0x4ddaf9 0x4ddae7 0x5d6b09 0x5e6c65 0x4ba65a 0x68a89c 0x68a86e 0x749f4d 0x4711c1 # 0x46b6c8 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dc711 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4ddaf8 internal/poll.(*pollDesc).waitRead+0x298 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4ddae6 internal/poll.(*FD).Read+0x286 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:167 # 0x5d6b08 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e6c64 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x4ba659 io.ReadAtLeast+0x99 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:332 # 0x68a89b io.ReadFull+0x9b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:351 # 0x68a86d github.com/couchbase/gomemcached.(*MCRequest).Receive+0x6d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_req.go:643 # 0x749f4c github.com/couchbase/gomemcached/client.(*UprFeed).runFeed+0x14c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/upr_feed.go:904 72 @ 0x43e876 0x44ee3e 0x926e45 0x4711c1 # 0x926e44 github.com/couchbase/goxdcr/parts.(*DcpNozzle).checkInactiveUprStreams+0x124 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1791 72 @ 0x43e876 0x44ee3e 0x927c71 0x4711c1 # 0x927c70 github.com/couchbase/goxdcr/parts.(*DcpNozzle).collectDcpDataChanLen+0x110 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1948 72 @ 0x43e876 0x44ee3e 0x938dd4 0x93e7fa 0x93e3e5 0x4711c1 # 0x938dd3 github.com/couchbase/goxdcr/parts.(*requestBuffer).flowControl+0x73 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:237 # 0x93e7f9 github.com/couchbase/goxdcr/parts.(*XmemNozzle).processBatch+0x39 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1120 # 0x93e3e4 github.com/couchbase/goxdcr/parts.(*XmemNozzle).processData_sendbatch+0x4e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1077 72 @ 0x43e876 0x44ee3e 0x94e93b 0x4711c1 # 0x94e93a github.com/couchbase/goxdcr/parts.(*XmemNozzle).selfMonitor+0x1da /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2912 67 @ 0x43e876 0x44ee3e 0x953c0b 0x93d9a6 0x93cf76 0x90e9d5 0x921162 0x4711c1 # 0x953c0a github.com/couchbase/goxdcr/parts.(*XmemNozzle).writeToDataChan+0xea /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3664 # 0x93d9a5 github.com/couchbase/goxdcr/parts.(*XmemNozzle).accumuBatch+0x3e5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:955 # 0x93cf75 github.com/couchbase/goxdcr/parts.(*XmemNozzle).Receive+0x115 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:907 # 0x90e9d4 github.com/couchbase/goxdcr/connector.(*Router).Forward+0x294 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/connector/router.go:118 # 0x921161 github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0xcc1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1102 64 @ 0x43e876 0x44ee3e 0xd9ff50 0x4711c1 # 0xd9ff4f github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).run.func2+0x18f /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1169 55 @ 0x43e876 0x44ee3e 0x5d8a26 0x4711c1 # 0x5d8a25 net.(*netFD).connect.func2+0x85 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_unix.go:118 50 @ 0x43e876 0x44ee3e 0x8ed86e 0x4711c1 # 0x8ed86d github.com/couchbase/goxdcr/peerToPeer.(*SendOpts).GetResults.func1+0x16d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:444 47 @ 0x43e876 0x44ff2f 0x44ff06 0x46cee7 0x48de4b 0x8ed55b 0x8ee20f 0xdd1151 0x9ca8c9 0x9c934d 0xa2e618 0xa38825 0xa36cc9 0x48c6e2 0xa366c7 0xa36695 0x4711c1 # 0x46cee6 sync.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:62 # 0x48de4a sync.(*WaitGroup).Wait+0x4a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/waitgroup.go:116 # 0x8ed55a github.com/couchbase/goxdcr/peerToPeer.(*SendOpts).GetResults+0x3ba /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:468 # 0x8ee20e github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).CheckVBMaster+0x32e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:549 # 0xdd1150 github.com/couchbase/goxdcr/factory.(*XDCRFactory).PreReplicationVBMasterCheck+0x2b0 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/factory/xdcr_factory.go:933 # 0x9ca8c8 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).runP2PProtocol+0xe8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:376 # 0x9c934c github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).Start+0x76c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:274 # 0xa2e617 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineManager).StartPipeline+0x4d7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:464 # 0xa38824 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).update+0x424 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1899 # 0xa36cc8 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run.func1+0x5c8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1610 # 0x48c6e1 sync.(*Once).doSlow+0xc1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:74 # 0xa366c6 sync.(*Once).Do+0x46 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:65 # 0xa36694 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run+0x14 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1588 46 @ 0x43e876 0x44ee3e 0x9b98ca 0x4711c1 # 0x9b98c9 github.com/couchbase/goxdcr/gen_server.(*GenServer).run+0xc9 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/gen_server/gen_server.go:83 46 @ 0x43e876 0x44ff2f 0x44ff06 0x46cee7 0x48de4b 0x9e7b16 0xdd0be5 0x4711c1 # 0x46cee6 sync.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:62 # 0x48de4a sync.(*WaitGroup).Wait+0x4a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/waitgroup.go:116 # 0x9e7b15 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).SetVBTimestamps+0xe55 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1162 # 0xdd0be4 github.com/couchbase/goxdcr/factory.(*XDCRFactory).SetStartSeqno+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/factory/xdcr_factory.go:888 45 @ 0x43e876 0x44ee3e 0x9d2d45 0x4711c1 # 0x9d2d44 github.com/couchbase/goxdcr/supervisor.(*GenericSupervisor).supervising+0x184 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/supervisor/generic_supervisor.go:207 44 @ 0x43e876 0x44ee3e 0x9dd4f8 0x4711c1 # 0x9dd4f7 github.com/couchbase/goxdcr/pipeline_svc.(*BandwidthThrottler).update+0x237 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/bandwidth_throttler.go:135 44 @ 0x43e876 0x44ee3e 0xa008e9 0x4711c1 # 0xa008e8 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).periodicMergerImpl+0xa8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:3516 44 @ 0x43e876 0x44ee3e 0xa0a7e5 0x4711c1 # 0xa0a7e4 github.com/couchbase/goxdcr/pipeline_svc.(*PipelineSupervisor).monitorPipelineHealth.func1+0x124 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/pipeline_supervisor.go:196 44 @ 0x43e876 0x44ee3e 0xa0b188 0x4711c1 # 0xa0b187 github.com/couchbase/goxdcr/pipeline_svc.(*PipelineSupervisor).checkAndLogFilterErrors+0x1a7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/pipeline_supervisor.go:273 44 @ 0x43e876 0x44ee3e 0xa0e125 0x4711c1 # 0xa0e124 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).logStats+0x224 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:332 44 @ 0x43e876 0x44ee3e 0xa12ad9 0x4711c1 # 0xa12ad8 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).subscribeToBucketTopologySvc+0x158 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:954 44 @ 0x43e876 0x44ee3e 0xa13885 0x4711c1 # 0xa13884 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).watchNotificationCh+0x144 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:1041 41 @ 0x43e876 0x44ee3e 0x9ed170 0x4711c1 # 0x9ed16f github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).checkpointing+0x40f /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1657 41 @ 0x43e876 0x44ee3e 0xa24297 0x4711c1 # 0xa24296 github.com/couchbase/goxdcr/pipeline_svc.(*TopologyChangeDetectorSvc).monitorSource.func1+0xd6 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/topology_change_detector.go:483 41 @ 0x43e876 0x44ee3e 0xa24e34 0x4711c1 # 0xa24e33 github.com/couchbase/goxdcr/pipeline_svc.(*TopologyChangeDetectorSvc).monitorTarget.func1+0xd3 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/topology_change_detector.go:572 38 @ 0x43e876 0x437117 0x46b6c9 0x4dc712 0x5d8539 0x5d851d 0x5d8541 0x5ebec5 0x5eb9f2 0x5e2f58 0x5ef68f 0x5ef529 0x5cdd52 0x5cd6a5 0x5ccd13 0x5cc72c 0x720447 0x72334c 0x721d90 0x4711c1 # 0x46b6c8 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dc711 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x5d8538 internal/poll.(*pollDesc).waitWrite+0x798 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:93 # 0x5d851c internal/poll.(*FD).WaitWrite+0x77c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:741 # 0x5d8540 net.(*netFD).connect+0x7a0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_unix.go:141 # 0x5ebec4 net.(*netFD).dial+0x3c4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/sock_posix.go:151 # 0x5eb9f1 net.socket+0x2d1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/sock_posix.go:70 # 0x5e2f57 net.internetSocket+0xf7 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/ipsock_posix.go:142 # 0x5ef68e net.(*sysDialer).doDialTCP+0xee /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/tcpsock_posix.go:74 # 0x5ef528 net.(*sysDialer).dialTCP+0x68 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/tcpsock_posix.go:64 # 0x5cdd51 net.(*sysDialer).dialSingle+0x1f1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:580 # 0x5cd6a4 net.(*sysDialer).dialSerial+0x244 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:548 # 0x5ccd12 net.(*sysDialer).dialParallel+0x412 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:449 # 0x5cc72b net.(*Dialer).DialContext+0x70b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:440 # 0x720446 net/http.(*Transport).dial+0xe6 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:1176 # 0x72334b net/http.(*Transport).dialConn+0x82b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:1614 # 0x721d8f net/http.(*Transport).dialConnFor+0xaf /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:1456 38 @ 0x43e876 0x44ee3e 0xa3691c 0x48c6e2 0xa366c7 0xa36695 0x4711c1 # 0xa3691b github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run.func1+0x21b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1599 # 0x48c6e1 sync.(*Once).doSlow+0xc1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:74 # 0xa366c6 sync.(*Once).Do+0x46 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:65 # 0xa36694 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run+0x14 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1588 37 @ 0x43e876 0x44ee3e 0x721106 0x71d01e 0x705219 0x6c5d37 0x6c55bb 0x6c78bb 0x8d87d3 0x8d87cb 0x8d943e 0xda80c9 0xda68e5 0x9eacbd 0x9e9c4c 0x4711c1 # 0x721105 net/http.(*Transport).getConn+0x5c5 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:1382 # 0x71d01d net/http.(*Transport).roundTrip+0x79d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:590 # 0x705218 net/http.(*Transport).RoundTrip+0x18 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/roundtrip.go:17 # 0x6c5d36 net/http.send+0x5f6 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:252 # 0x6c55ba net/http.(*Client).send+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:176 # 0x6c78ba net/http.(*Client).do+0x8fa /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:716 # 0x8d87d2 net/http.(*Client).Do+0x52 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:582 # 0x8d87ca github.com/couchbase/goxdcr/utils.(*Utilities).doRestCall+0x4a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2234 # 0x8d943d github.com/couchbase/goxdcr/utils.(*Utilities).InvokeRestWithRetryWithAuth+0x37d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2321 # 0xda80c8 github.com/couchbase/goxdcr/service_impl.(*CAPIService).send_post+0x308 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/capi_service.go:291 # 0xda68e4 github.com/couchbase/goxdcr/service_impl.(*CAPIService).PreReplicate+0x144 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/capi_service.go:122 # 0x9eacbc github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).getDataFromCkpts+0x4bc /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1442 # 0x9e9c4b github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).startSeqnoGetter+0x2cb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1319 36 @ 0x43e876 0x44ee3e 0x923b65 0x4711c1 # 0x923b64 github.com/couchbase/goxdcr/parts.(*DcpNozzle).startUprStreams+0x224 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1344 36 @ 0x43e876 0x44ee3e 0xa13425 0xa13435 0xa109c2 0xa1088f 0xa0fc33 0xa0f40f 0xa0e525 0xa0da65 0x4711c1 # 0xa13424 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).getNotification+0xc4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:1029 # 0xa13434 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).subscribeToBucketTopologySvc.func1.2+0xd4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:975 # 0xa109c1 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).calculateChangesLeftAndDocsProcessedMainPipeline+0x101 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:670 # 0xa1088e github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).calculateChangesLeftAndDocProcessed+0x10e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:654 # 0xa0fc32 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).processCalculatedStats+0x72 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:548 # 0xa0f40e github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).processRawStats+0x7ee /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:528 # 0xa0e524 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).updateStatsOnce+0x144 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:355 # 0xa0da64 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).updateStats+0x3e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:309 30 @ 0x43e876 0x44ff2f 0x44ff06 0x46cee7 0x48de4b 0x93e9bf 0x4711c1 # 0x46cee6 sync.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:62 # 0x48de4a sync.(*WaitGroup).Wait+0x4a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/waitgroup.go:116 # 0x93e9be github.com/couchbase/goxdcr/parts.(*XmemNozzle).finalCleanup+0x3e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1137 28 @ 0x43e876 0x44ee3e 0x729154 0x71d07a 0x705219 0x6c5d37 0x6c55bb 0x6c78bb 0x8d87d3 0x8d87cb 0x8d7a53 0x8e9022 0x8ec8f7 0x4711c1 # 0x729153 net/http.(*persistConn).roundTrip+0x993 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:2638 # 0x71d079 net/http.(*Transport).roundTrip+0x7f9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:603 # 0x705218 net/http.(*Transport).RoundTrip+0x18 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/roundtrip.go:17 # 0x6c5d36 net/http.send+0x5f6 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:252 # 0x6c55ba net/http.(*Client).send+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:176 # 0x6c78ba net/http.(*Client).do+0x8fa /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:716 # 0x8d87d2 net/http.(*Client).Do+0x52 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:582 # 0x8d87ca github.com/couchbase/goxdcr/utils.(*Utilities).doRestCall+0x4a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2234 # 0x8d7a52 github.com/couchbase/goxdcr/utils.(*Utilities).QueryRestApiWithAuth+0x1b2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2096 # 0x8e9021 github.com/couchbase/goxdcr/peerToPeer.(*P2pCommAPIimpl).P2PSend+0x241 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerCommAPI.go:133 # 0x8ec8f6 github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).sendToSpecifiedPeersOnce.func1.1+0x316 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:309 23 @ 0x43e876 0x44ff2f 0x44ff06 0x46cee7 0x48de4b 0x8ec406 0x8dba7b 0x8db83b 0x8eb929 0x8eb59d 0x8ee1ed 0xdd1151 0x9ca8c9 0x9c934d 0xa2e618 0xa38825 0xa36cc9 0x48c6e2 0xa366c7 0xa36695 0x4711c1 # 0x46cee6 sync.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:62 # 0x48de4a sync.(*WaitGroup).Wait+0x4a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/waitgroup.go:116 # 0x8ec405 github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).sendToSpecifiedPeersOnce.func1+0x3e5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:344 # 0x8dba7a github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithOriginalError+0xba /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2587 # 0x8db83a github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutor+0x3a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2575 # 0x8eb928 github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).sendToSpecifiedPeersOnce+0x328 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:356 # 0x8eb59c github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).sendToEachPeerOnce+0xfc /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:257 # 0x8ee1ec github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).CheckVBMaster+0x30c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:541 # 0xdd1150 github.com/couchbase/goxdcr/factory.(*XDCRFactory).PreReplicationVBMasterCheck+0x2b0 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/factory/xdcr_factory.go:933 # 0x9ca8c8 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).runP2PProtocol+0xe8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:376 # 0x9c934c github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).Start+0x76c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:274 # 0xa2e617 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineManager).StartPipeline+0x4d7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:464 # 0xa38824 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).update+0x424 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1899 # 0xa36cc8 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run.func1+0x5c8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1610 # 0x48c6e1 sync.(*Once).doSlow+0xc1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:74 # 0xa366c6 sync.(*Once).Do+0x46 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:65 # 0xa36694 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run+0x14 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1588 22 @ 0x43e876 0x40ba5d 0x40b558 0xd9fc51 0x4711c1 # 0xd9fc50 github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).run+0x350 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1186 21 @ 0x43e876 0x437117 0x46b6c9 0x4dc712 0x4ddaf9 0x4ddae7 0x5d6b09 0x5e6c65 0x706d71 0x69251f 0x69267d 0x70cd3c 0x4711c1 # 0x46b6c8 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dc711 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4ddaf8 internal/poll.(*pollDesc).waitRead+0x298 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4ddae6 internal/poll.(*FD).Read+0x286 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:167 # 0x5d6b08 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e6c64 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x706d70 net/http.(*connReader).Read+0x170 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/server.go:782 # 0x69251e bufio.(*Reader).fill+0xfe /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/bufio/bufio.go:106 # 0x69267c bufio.(*Reader).Peek+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/bufio/bufio.go:144 # 0x70cd3b net/http.(*conn).serve+0x77b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/server.go:2030 11 @ 0x43e876 0x44ee3e 0xda0065 0x4711c1 # 0xda0064 github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).run.func1+0x84 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1143 11 @ 0x43e876 0x44ff2f 0x44ff06 0x46cee7 0x48de4b 0x9c7c47 0x7e6842 0x4711c1 # 0x46cee6 sync.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:62 # 0x48de4a sync.(*WaitGroup).Wait+0x4a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/waitgroup.go:116 # 0x9c7c46 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPartsWithTimeout.func1+0x226 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:161 # 0x7e6841 github.com/couchbase/goxdcr/base.ExecWithTimeout.func1+0x21 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/simple_utils.go:48 11 @ 0x43e876 0x44ff2f 0x44ff06 0x46cee7 0x48de4b 0x9c8954 0x9c811f 0x4711c1 # 0x46cee6 sync.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:62 # 0x48de4a sync.(*WaitGroup).Wait+0x4a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/waitgroup.go:116 # 0x9c8953 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart+0x5d3 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:214 # 0x9c811e github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPartsWithTimeout.func1.1+0xde /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:153 8 @ 0x43e876 0x46dfb5 0x8d9232 0xda80c9 0xda68e5 0x9eacbd 0x9e9c4c 0x4711c1 # 0x46dfb4 time.Sleep+0x134 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/time.go:195 # 0x8d9231 github.com/couchbase/goxdcr/utils.(*Utilities).InvokeRestWithRetryWithAuth+0x171 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2341 # 0xda80c8 github.com/couchbase/goxdcr/service_impl.(*CAPIService).send_post+0x308 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/capi_service.go:291 # 0xda68e4 github.com/couchbase/goxdcr/service_impl.(*CAPIService).PreReplicate+0x144 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/capi_service.go:122 # 0x9eacbc github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).getDataFromCkpts+0x4bc /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1442 # 0x9e9c4b github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).startSeqnoGetter+0x2cb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1319 7 @ 0x43e876 0x437117 0x46b6c9 0x4dc712 0x4ddaf9 0x4ddae7 0x5d6b09 0x5e6c65 0x7257ce 0x69251f 0x69310f 0x6c2785 0x6c21e8 0x6c24b6 0x71a65c 0x71a525 0x729ec2 0x681ba8 0x68179b 0x6813d8 0x95d6a5 0x4711c1 # 0x46b6c8 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dc711 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4ddaf8 internal/poll.(*pollDesc).waitRead+0x298 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4ddae6 internal/poll.(*FD).Read+0x286 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:167 # 0x5d6b08 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e6c64 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x7257cd net/http.(*persistConn).Read+0x4d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:1943 # 0x69251e bufio.(*Reader).fill+0xfe /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/bufio/bufio.go:106 # 0x69310e bufio.(*Reader).ReadSlice+0x2e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/bufio/bufio.go:372 # 0x6c2784 net/http/internal.readChunkLine+0x24 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/internal/chunked.go:129 # 0x6c21e7 net/http/internal.(*chunkedReader).beginChunk+0x27 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/internal/chunked.go:48 # 0x6c24b5 net/http/internal.(*chunkedReader).Read+0x155 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/internal/chunked.go:98 # 0x71a65b net/http.(*body).readLocked+0x3b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transfer.go:839 # 0x71a524 net/http.(*body).Read+0x124 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transfer.go:831 # 0x729ec1 net/http.(*bodyEOFSignal).Read+0x141 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:2792 # 0x681ba7 encoding/json.(*Decoder).refill+0x187 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/encoding/json/stream.go:165 # 0x68179a encoding/json.(*Decoder).readValue+0xba /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/encoding/json/stream.go:140 # 0x6813d7 encoding/json.(*Decoder).Decode+0x77 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/encoding/json/stream.go:63 # 0x95d6a4 github.com/couchbase/cbauth/metakv.doRunObserveChildren.func1+0xc4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:284 7 @ 0x43e876 0x437117 0x46b6c9 0x4dc712 0x5d8539 0x5d851d 0x5d8541 0x5ebec5 0x5eb9f2 0x5e2f58 0x5ef68f 0x5ef529 0x5cdd52 0x5cd6a5 0x5ccd13 0x5cc72c 0x5cbfc7 0x5cbfc8 0x73b8ea 0x73b96e 0x7e27a8 0x7df455 0x7df3db 0x954be2 0x8dbd4a 0x954705 0x952d8c 0x94ff17 0x939314 0x94fa89 0x4711c1 # 0x46b6c8 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dc711 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x5d8538 internal/poll.(*pollDesc).waitWrite+0x798 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:93 # 0x5d851c internal/poll.(*FD).WaitWrite+0x77c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:741 # 0x5d8540 net.(*netFD).connect+0x7a0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_unix.go:141 # 0x5ebec4 net.(*netFD).dial+0x3c4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/sock_posix.go:151 # 0x5eb9f1 net.socket+0x2d1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/sock_posix.go:70 # 0x5e2f57 net.internetSocket+0xf7 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/ipsock_posix.go:142 # 0x5ef68e net.(*sysDialer).doDialTCP+0xee /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/tcpsock_posix.go:74 # 0x5ef528 net.(*sysDialer).dialTCP+0x68 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/tcpsock_posix.go:64 # 0x5cdd51 net.(*sysDialer).dialSingle+0x1f1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:580 # 0x5cd6a4 net.(*sysDialer).dialSerial+0x244 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:548 # 0x5ccd12 net.(*sysDialer).dialParallel+0x412 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:449 # 0x5cc72b net.(*Dialer).DialContext+0x70b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:440 # 0x5cbfc6 net.(*Dialer).Dial+0x86 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:367 # 0x5cbfc7 net.DialTimeout+0x87 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:349 # 0x73b8e9 github.com/couchbase/gomemcached/client.glob..func1+0x29 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:189 # 0x73b96d github.com/couchbase/gomemcached/client.Connect+0x2d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:195 # 0x7e27a7 github.com/couchbase/goxdcr/base.NewConn+0xa7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:605 # 0x7df454 github.com/couchbase/goxdcr/base.(*connPool).newConn+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:208 # 0x7df3da github.com/couchbase/goxdcr/base.(*connPool).GetNew+0x1a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:204 # 0x954be1 github.com/couchbase/goxdcr/parts.(*XmemNozzle).getClientWithRetry.func1+0x2a1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3774 # 0x8dbd49 github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithFinishSignal+0x109 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2615 # 0x954704 github.com/couchbase/goxdcr/parts.(*XmemNozzle).getClientWithRetry+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3777 # 0x952d8b github.com/couchbase/goxdcr/parts.(*XmemNozzle).repairConn+0x1cb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3550 # 0x94ff16 github.com/couchbase/goxdcr/parts.(*XmemNozzle).resendIfTimeout+0x376 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3036 # 0x939313 github.com/couchbase/goxdcr/parts.(*requestBuffer).modSlot+0x53 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:296 # 0x94fa88 github.com/couchbase/goxdcr/parts.(*XmemNozzle).checkAndRepairBufferMonitor+0x348 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2995 7 @ 0x43e876 0x44ee3e 0x7271c5 0x4711c1 # 0x7271c4 net/http.(*persistConn).readLoop+0xd84 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:2227 5 @ 0x43e876 0x437117 0x46b6c9 0x4dc712 0x5d8539 0x5d851d 0x5d8541 0x5ebec5 0x5eb9f2 0x5e2f58 0x5ef68f 0x5ef529 0x5cdd52 0x5cd6a5 0x5ccd13 0x5cc72c 0x5cbfc7 0x5cbfc8 0x73b8ea 0x73b96e 0x7e27a8 0x7df455 0x7df3db 0x954be2 0x8dbd4a 0x954705 0x94a38e 0x94b32c 0x93bf0b 0x9c86a2 0x9c8afa 0x4711c1 # 0x46b6c8 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dc711 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x5d8538 internal/poll.(*pollDesc).waitWrite+0x798 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:93 # 0x5d851c internal/poll.(*FD).WaitWrite+0x77c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:741 # 0x5d8540 net.(*netFD).connect+0x7a0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_unix.go:141 # 0x5ebec4 net.(*netFD).dial+0x3c4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/sock_posix.go:151 # 0x5eb9f1 net.socket+0x2d1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/sock_posix.go:70 # 0x5e2f57 net.internetSocket+0xf7 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/ipsock_posix.go:142 # 0x5ef68e net.(*sysDialer).doDialTCP+0xee /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/tcpsock_posix.go:74 # 0x5ef528 net.(*sysDialer).dialTCP+0x68 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/tcpsock_posix.go:64 # 0x5cdd51 net.(*sysDialer).dialSingle+0x1f1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:580 # 0x5cd6a4 net.(*sysDialer).dialSerial+0x244 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:548 # 0x5ccd12 net.(*sysDialer).dialParallel+0x412 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:449 # 0x5cc72b net.(*Dialer).DialContext+0x70b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:440 # 0x5cbfc6 net.(*Dialer).Dial+0x86 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:367 # 0x5cbfc7 net.DialTimeout+0x87 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:349 # 0x73b8e9 github.com/couchbase/gomemcached/client.glob..func1+0x29 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:189 # 0x73b96d github.com/couchbase/gomemcached/client.Connect+0x2d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:195 # 0x7e27a7 github.com/couchbase/goxdcr/base.NewConn+0xa7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:605 # 0x7df454 github.com/couchbase/goxdcr/base.(*connPool).newConn+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:208 # 0x7df3da github.com/couchbase/goxdcr/base.(*connPool).GetNew+0x1a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:204 # 0x954be1 github.com/couchbase/goxdcr/parts.(*XmemNozzle).getClientWithRetry.func1+0x2a1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3774 # 0x8dbd49 github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithFinishSignal+0x109 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2615 # 0x954704 github.com/couchbase/goxdcr/parts.(*XmemNozzle).getClientWithRetry+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3777 # 0x94a38d github.com/couchbase/goxdcr/parts.(*XmemNozzle).initializeConnection+0x38d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2406 # 0x94b32b github.com/couchbase/goxdcr/parts.(*XmemNozzle).initialize+0x72b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2550 # 0x93bf0a github.com/couchbase/goxdcr/parts.(*XmemNozzle).Start+0x1ca /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:807 # 0x9c86a1 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart+0x321 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:233 # 0x9c8af9 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart.func1+0xb9 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:209 5 @ 0x43e876 0x437117 0x46b6c9 0x4dc712 0x5d8539 0x5d851d 0x5d8541 0x5ebec5 0x5eb9f2 0x5e2f58 0x5ef68f 0x5ef529 0x5cdd52 0x5cd6a5 0x5ccd13 0x5cc72c 0x5cbfc7 0x5cbfc8 0x73b8ea 0x73b96e 0x7e27a8 0x8cbbb9 0x8cc0e5 0x8dba7b 0x8db83b 0x8cbec8 0x8cc43f 0x9e32de 0x9e25cf 0x48c6e2 0x9e2270 0x9e223a # 0x46b6c8 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dc711 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x5d8538 internal/poll.(*pollDesc).waitWrite+0x798 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:93 # 0x5d851c internal/poll.(*FD).WaitWrite+0x77c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:741 # 0x5d8540 net.(*netFD).connect+0x7a0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_unix.go:141 # 0x5ebec4 net.(*netFD).dial+0x3c4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/sock_posix.go:151 # 0x5eb9f1 net.socket+0x2d1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/sock_posix.go:70 # 0x5e2f57 net.internetSocket+0xf7 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/ipsock_posix.go:142 # 0x5ef68e net.(*sysDialer).doDialTCP+0xee /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/tcpsock_posix.go:74 # 0x5ef528 net.(*sysDialer).dialTCP+0x68 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/tcpsock_posix.go:64 # 0x5cdd51 net.(*sysDialer).dialSingle+0x1f1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:580 # 0x5cd6a4 net.(*sysDialer).dialSerial+0x244 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:548 # 0x5ccd12 net.(*sysDialer).dialParallel+0x412 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:449 # 0x5cc72b net.(*Dialer).DialContext+0x70b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:440 # 0x5cbfc6 net.(*Dialer).Dial+0x86 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:367 # 0x5cbfc7 net.DialTimeout+0x87 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:349 # 0x73b8e9 github.com/couchbase/gomemcached/client.glob..func1+0x29 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:189 # 0x73b96d github.com/couchbase/gomemcached/client.Connect+0x2d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:195 # 0x7e27a7 github.com/couchbase/goxdcr/base.NewConn+0xa7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:605 # 0x8cbbb8 github.com/couchbase/goxdcr/utils.(*Utilities).GetMemcachedRawConn+0x1f8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:524 # 0x8cc0e4 github.com/couchbase/goxdcr/utils.(*Utilities).GetRemoteMemcachedConnectionWFeatures.func1+0x104 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:538 # 0x8dba7a github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithOriginalError+0xba /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2587 # 0x8db83a github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutor+0x3a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2575 # 0x8cbec7 github.com/couchbase/goxdcr/utils.(*Utilities).GetRemoteMemcachedConnectionWFeatures+0x1c7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:554 # 0x8cc43e github.com/couchbase/goxdcr/utils.(*Utilities).GetRemoteMemcachedConnection+0x9e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:568 # 0x9e32dd github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).getNewMemcachedClient+0x49d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:708 # 0x9e25ce github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).initConnections.func1+0x30e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:621 # 0x48c6e1 sync.(*Once).doSlow+0xc1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:74 # 0x9e226f sync.(*Once).Do+0x4f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:65 # 0x9e2239 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).initConnections+0x19 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:609 5 @ 0x43e876 0x44ee3e 0x92077f 0x4711c1 # 0x92077e github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0x2de /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:995 5 @ 0x43e876 0x44ff2f 0x44ff06 0x46cee7 0x48de4b 0x8ec406 0x8dba7b 0x8db83b 0x8eb929 0x8eb59d 0x8ee1ed 0xdd1151 0x9ca8c9 0x9c934d 0xa2e618 0xa38825 0xa37cfb 0xa36ec6 0x48c6e2 0xa366c7 0xa36695 0x4711c1 # 0x46cee6 sync.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:62 # 0x48de4a sync.(*WaitGroup).Wait+0x4a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/waitgroup.go:116 # 0x8ec405 github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).sendToSpecifiedPeersOnce.func1+0x3e5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:344 # 0x8dba7a github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithOriginalError+0xba /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2587 # 0x8db83a github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutor+0x3a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2575 # 0x8eb928 github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).sendToSpecifiedPeersOnce+0x328 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:356 # 0x8eb59c github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).sendToEachPeerOnce+0xfc /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:257 # 0x8ee1ec github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).CheckVBMaster+0x30c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:541 # 0xdd1150 github.com/couchbase/goxdcr/factory.(*XDCRFactory).PreReplicationVBMasterCheck+0x2b0 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/factory/xdcr_factory.go:933 # 0x9ca8c8 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).runP2PProtocol+0xe8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:376 # 0x9c934c github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).Start+0x76c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:274 # 0xa2e617 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineManager).StartPipeline+0x4d7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:464 # 0xa38824 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).update+0x424 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1899 # 0xa37cfa github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).checkIfNeedToReUpdate+0x11a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1739 # 0xa36ec5 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run.func1+0x7c5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1621 # 0x48c6e1 sync.(*Once).doSlow+0xc1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:74 # 0xa366c6 sync.(*Once).Do+0x46 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:65 # 0xa36694 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run+0x14 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1588 4 @ 0x43e876 0x44ee3e 0x95d269 0x95d7b8 0x95d77a 0xde3688 0x4711c1 # 0x95d268 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x428 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:301 # 0x95d7b7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:259 # 0x95d779 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x19 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:389 # 0xde3687 github.com/couchbase/goxdcr/replication_manager.(*MetakvChangeListener).observeChildren+0x87 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/replication_manager/metakv_change_listener.go:171 4 @ 0x43e876 0x44ee3e 0xa0d973 0x4711c1 # 0xa0d972 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).updateStats+0x2f2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:292 3 @ 0x43e876 0x437117 0x46b6c9 0x4dc712 0x4ddaf9 0x4ddae7 0x5d6b09 0x5e6c65 0x4ba65a 0x68b539 0x68b503 0x744545 0x74450d 0x73bf49 0x73edb2 0x7df953 0x7e28ed 0x7df455 0x7df3db 0x954be2 0x8dbd4a 0x954705 0x94a38e 0x94b32c 0x93bf0b 0x9c86a2 0x9c8afa 0x4711c1 # 0x46b6c8 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dc711 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4ddaf8 internal/poll.(*pollDesc).waitRead+0x298 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4ddae6 internal/poll.(*FD).Read+0x286 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:167 # 0x5d6b08 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e6c64 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x4ba659 io.ReadAtLeast+0x99 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:332 # 0x68b538 io.ReadFull+0xd8 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:351 # 0x68b502 github.com/couchbase/gomemcached.(*MCResponse).ReceiveWithBuf+0xa2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:169 # 0x744544 github.com/couchbase/gomemcached.(*MCResponse).Receive+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:156 # 0x74450c github.com/couchbase/gomemcached/client.getResponse+0x6c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/transport.go:35 # 0x73bf48 github.com/couchbase/gomemcached/client.(*Client).Send+0x68 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:280 # 0x73edb1 github.com/couchbase/gomemcached/client.(*Client).SelectBucket+0x91 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:767 # 0x7df952 github.com/couchbase/goxdcr/base.authClient+0x212 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:251 # 0x7e28ec github.com/couchbase/goxdcr/base.NewConn+0x1ec /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:612 # 0x7df454 github.com/couchbase/goxdcr/base.(*connPool).newConn+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:208 # 0x7df3da github.com/couchbase/goxdcr/base.(*connPool).GetNew+0x1a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:204 # 0x954be1 github.com/couchbase/goxdcr/parts.(*XmemNozzle).getClientWithRetry.func1+0x2a1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3774 # 0x8dbd49 github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithFinishSignal+0x109 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2615 # 0x954704 github.com/couchbase/goxdcr/parts.(*XmemNozzle).getClientWithRetry+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3777 # 0x94a38d github.com/couchbase/goxdcr/parts.(*XmemNozzle).initializeConnection+0x38d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2406 # 0x94b32b github.com/couchbase/goxdcr/parts.(*XmemNozzle).initialize+0x72b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2550 # 0x93bf0a github.com/couchbase/goxdcr/parts.(*XmemNozzle).Start+0x1ca /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:807 # 0x9c86a1 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart+0x321 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:233 # 0x9c8af9 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart.func1+0xb9 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:209 3 @ 0x43e876 0x44ee3e 0x7e663f 0x9c75dd 0x9c9b13 0xa2e618 0xa38825 0xa36cc9 0x48c6e2 0xa366c7 0xa36695 0x4711c1 # 0x7e663e github.com/couchbase/goxdcr/base.ExecWithTimeout+0x19e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/simple_utils.go:56 # 0x9c75dc github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPartsWithTimeout+0x13c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:175 # 0x9c9b12 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).Start+0xf32 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:326 # 0xa2e617 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineManager).StartPipeline+0x4d7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:464 # 0xa38824 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).update+0x424 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1899 # 0xa36cc8 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run.func1+0x5c8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1610 # 0x48c6e1 sync.(*Once).doSlow+0xc1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:74 # 0xa366c6 sync.(*Once).Do+0x46 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:65 # 0xa36694 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run+0x14 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1588 3 @ 0x43e876 0x44ee3e 0x7e663f 0x9e1288 0x4711c1 # 0x7e663e github.com/couchbase/goxdcr/base.ExecWithTimeout+0x19e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/simple_utils.go:56 # 0x9e1287 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).initConnBg+0x67 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:538 3 @ 0x43e876 0x44ee3e 0x8e789f 0x4711c1 # 0x8e789e github.com/couchbase/goxdcr/peerToPeer.(*HandlerCommon).ClearOpaqueBg+0x7e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/handlerCommon.go:148 3 @ 0x43e876 0x44ee3e 0x95d269 0x95d7b8 0x95d77a 0xd844fc 0xd7d524 0xd84447 0x4711c1 # 0x95d268 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x428 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:301 # 0x95d7b7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:259 # 0x95d779 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x19 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:389 # 0xd844fb github.com/couchbase/eventing-ee/evaluator/store.ObserveChanges.func1+0x7b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing-ee/evaluator/store/store.go:389 # 0xd7d523 github.com/couchbase/eventing-ee/evaluator/utils.ExponentialBackoffRetry+0x43 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing-ee/evaluator/utils/retry.go:57 # 0xd84446 github.com/couchbase/eventing-ee/evaluator/store.ObserveChanges+0x86 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing-ee/evaluator/store/store.go:387 3 @ 0x43e876 0x44ff2f 0x44ff06 0x46cee7 0x48de4b 0x8ed55b 0x8ee20f 0xdd1151 0x9ca8c9 0x9c934d 0xa2e618 0xa38825 0xa37cfb 0xa36ec6 0x48c6e2 0xa366c7 0xa36695 0x4711c1 # 0x46cee6 sync.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:62 # 0x48de4a sync.(*WaitGroup).Wait+0x4a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/waitgroup.go:116 # 0x8ed55a github.com/couchbase/goxdcr/peerToPeer.(*SendOpts).GetResults+0x3ba /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:468 # 0x8ee20e github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).CheckVBMaster+0x32e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:549 # 0xdd1150 github.com/couchbase/goxdcr/factory.(*XDCRFactory).PreReplicationVBMasterCheck+0x2b0 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/factory/xdcr_factory.go:933 # 0x9ca8c8 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).runP2PProtocol+0xe8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:376 # 0x9c934c github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).Start+0x76c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:274 # 0xa2e617 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineManager).StartPipeline+0x4d7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:464 # 0xa38824 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).update+0x424 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1899 # 0xa37cfa github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).checkIfNeedToReUpdate+0x11a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1739 # 0xa36ec5 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run.func1+0x7c5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1621 # 0x48c6e1 sync.(*Once).doSlow+0xc1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:74 # 0xa366c6 sync.(*Once).Do+0x46 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:65 # 0xa36694 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run+0x14 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1588 2 @ 0x43e876 0x437117 0x46b6c9 0x4dc712 0x4ddaf9 0x4ddae7 0x5d6b09 0x5e6c65 0x4ba65a 0x68b539 0x68b503 0x744545 0x74450d 0x73bf49 0x73e315 0x73e2f5 0x7df7b6 0x7e28ed 0x7df455 0x7df3db 0x954be2 0x8dbd4a 0x954705 0x94a38e 0x94b32c 0x93bf0b 0x9c86a2 0x9c8afa 0x4711c1 # 0x46b6c8 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dc711 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4ddaf8 internal/poll.(*pollDesc).waitRead+0x298 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4ddae6 internal/poll.(*FD).Read+0x286 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:167 # 0x5d6b08 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e6c64 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x4ba659 io.ReadAtLeast+0x99 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:332 # 0x68b538 io.ReadFull+0xd8 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:351 # 0x68b502 github.com/couchbase/gomemcached.(*MCResponse).ReceiveWithBuf+0xa2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:169 # 0x744544 github.com/couchbase/gomemcached.(*MCResponse).Receive+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:156 # 0x74450c github.com/couchbase/gomemcached/client.getResponse+0x6c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/transport.go:35 # 0x73bf48 github.com/couchbase/gomemcached/client.(*Client).Send+0x68 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:280 # 0x73e314 github.com/couchbase/gomemcached/client.(*Client).AuthList+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:674 # 0x73e2f4 github.com/couchbase/gomemcached/client.(*Client).Auth+0x34 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:680 # 0x7df7b5 github.com/couchbase/goxdcr/base.authClient+0x75 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:236 # 0x7e28ec github.com/couchbase/goxdcr/base.NewConn+0x1ec /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:612 # 0x7df454 github.com/couchbase/goxdcr/base.(*connPool).newConn+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:208 # 0x7df3da github.com/couchbase/goxdcr/base.(*connPool).GetNew+0x1a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:204 # 0x954be1 github.com/couchbase/goxdcr/parts.(*XmemNozzle).getClientWithRetry.func1+0x2a1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3774 # 0x8dbd49 github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithFinishSignal+0x109 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2615 # 0x954704 github.com/couchbase/goxdcr/parts.(*XmemNozzle).getClientWithRetry+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3777 # 0x94a38d github.com/couchbase/goxdcr/parts.(*XmemNozzle).initializeConnection+0x38d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2406 # 0x94b32b github.com/couchbase/goxdcr/parts.(*XmemNozzle).initialize+0x72b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2550 # 0x93bf0a github.com/couchbase/goxdcr/parts.(*XmemNozzle).Start+0x1ca /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:807 # 0x9c86a1 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart+0x321 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:233 # 0x9c8af9 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart.func1+0xb9 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:209 2 @ 0x43e876 0x437117 0x46b6c9 0x4dc712 0x4ddaf9 0x4ddae7 0x5d6b09 0x5e6c65 0x4ba65a 0x68b539 0x68b503 0x744545 0x74450d 0x74228e 0x9e4724 0x8dbd4a 0x9e404e 0x4711c1 # 0x46b6c8 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dc711 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4ddaf8 internal/poll.(*pollDesc).waitRead+0x298 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4ddae6 internal/poll.(*FD).Read+0x286 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:167 # 0x5d6b08 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e6c64 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x4ba659 io.ReadAtLeast+0x99 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:332 # 0x68b538 io.ReadFull+0xd8 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:351 # 0x68b502 github.com/couchbase/gomemcached.(*MCResponse).ReceiveWithBuf+0xa2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:169 # 0x744544 github.com/couchbase/gomemcached.(*MCResponse).Receive+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:156 # 0x74450c github.com/couchbase/gomemcached/client.getResponse+0x6c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/transport.go:35 # 0x74228d github.com/couchbase/gomemcached/client.(*Client).StatsMap+0x10d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:1426 # 0x9e4723 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).getTargetKVStatsMapWithRetry.func1+0x3e3 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:798 # 0x8dbd49 github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithFinishSignal+0x109 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2615 # 0x9e404d github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).getTargetKVStatsMapWithRetry+0x20d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:812 2 @ 0x43e876 0x44ff2f 0x44ff06 0x46cee7 0x48de4b 0x9e3acc 0x9ef956 0x9eef2a 0x4711c1 # 0x46cee6 sync.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:62 # 0x48de4a sync.(*WaitGroup).Wait+0x4a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/waitgroup.go:116 # 0x9e3acb github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).getHighSeqnoAndVBUuidFromTarget+0x3ab /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:759 # 0x9ef955 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).gatherCkptData+0xb5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1849 # 0x9eef29 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).performCkpt+0x409 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1813 2 @ 0x43e876 0x46dfb5 0x8dbb88 0x8db83b 0x8eb929 0x8eb59d 0x8ee1ed 0xdd1151 0x9ca8c9 0x9c934d 0xa2e618 0xa38825 0xa37cfb 0xa36ec6 0x48c6e2 0xa366c7 0xa36695 0x4711c1 # 0x46dfb4 time.Sleep+0x134 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/time.go:195 # 0x8dbb87 github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithOriginalError+0x1c7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2593 # 0x8db83a github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutor+0x3a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2575 # 0x8eb928 github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).sendToSpecifiedPeersOnce+0x328 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:356 # 0x8eb59c github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).sendToEachPeerOnce+0xfc /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:257 # 0x8ee1ec github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).CheckVBMaster+0x30c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:541 # 0xdd1150 github.com/couchbase/goxdcr/factory.(*XDCRFactory).PreReplicationVBMasterCheck+0x2b0 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/factory/xdcr_factory.go:933 # 0x9ca8c8 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).runP2PProtocol+0xe8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:376 # 0x9c934c github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).Start+0x76c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:274 # 0xa2e617 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineManager).StartPipeline+0x4d7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:464 # 0xa38824 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).update+0x424 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1899 # 0xa37cfa github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).checkIfNeedToReUpdate+0x11a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1739 # 0xa36ec5 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run.func1+0x7c5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1621 # 0x48c6e1 sync.(*Once).doSlow+0xc1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:74 # 0xa366c6 sync.(*Once).Do+0x46 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:65 # 0xa36694 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run+0x14 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1588 1 @ 0x433bd6 0x46b2a5 0x7d0fb5 0x7d0dcd 0x7cdb4b 0x7dce05 0x7dd953 0x70e0af 0x70fa49 0x7111f6 0x70cbd2 0x4711c1 # 0x46b2a4 runtime/pprof.runtime_goroutineProfileWithLabels+0x24 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/mprof.go:844 # 0x7d0fb4 runtime/pprof.writeRuntimeProfile+0xb4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/pprof/pprof.go:734 # 0x7d0dcc runtime/pprof.writeGoroutine+0x4c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/pprof/pprof.go:694 # 0x7cdb4a runtime/pprof.(*Profile).WriteTo+0x14a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/pprof/pprof.go:329 # 0x7dce04 net/http/pprof.handler.ServeHTTP+0x4a4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/pprof/pprof.go:259 # 0x7dd952 net/http/pprof.Index+0xf2 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/pprof/pprof.go:376 # 0x70e0ae net/http.HandlerFunc.ServeHTTP+0x2e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/server.go:2122 # 0x70fa48 net/http.(*ServeMux).ServeHTTP+0x148 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/server.go:2500 # 0x7111f5 net/http.serverHandler.ServeHTTP+0x315 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/server.go:2936 # 0x70cbd1 net/http.(*conn).serve+0x611 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/server.go:1995 1 @ 0x43e876 0x40ba5d 0x40b558 0xe028f3 0x43e447 0x4711c1 # 0xe028f2 main.main+0x14d2 goproj/src/github.com/couchbase/goxdcr/main/main.go:275 # 0x43e446 runtime.main+0x206 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/proc.go:250 1 @ 0x43e876 0x437117 0x46b6c9 0x4dc712 0x4ddaf9 0x4ddae7 0x5d6b09 0x5e6c65 0x4ba65a 0x68b539 0x68b503 0x744545 0x74450d 0x73bf49 0x73ecbd 0x73e36c 0x7df7b6 0x7e28ed 0x7df455 0x7df3db 0x954be2 0x8dbd4a 0x954705 0x94a38e 0x94b32c 0x93bf0b 0x9c86a2 0x9c8afa 0x4711c1 # 0x46b6c8 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dc711 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4ddaf8 internal/poll.(*pollDesc).waitRead+0x298 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4ddae6 internal/poll.(*FD).Read+0x286 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:167 # 0x5d6b08 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e6c64 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x4ba659 io.ReadAtLeast+0x99 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:332 # 0x68b538 io.ReadFull+0xd8 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:351 # 0x68b502 github.com/couchbase/gomemcached.(*MCResponse).ReceiveWithBuf+0xa2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:169 # 0x744544 github.com/couchbase/gomemcached.(*MCResponse).Receive+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:156 # 0x74450c github.com/couchbase/gomemcached/client.getResponse+0x6c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/transport.go:35 # 0x73bf48 github.com/couchbase/gomemcached/client.(*Client).Send+0x68 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:280 # 0x73ecbc github.com/couchbase/gomemcached/client.(*Client).AuthPlain+0x23c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:759 # 0x73e36b github.com/couchbase/gomemcached/client.(*Client).Auth+0xab /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:688 # 0x7df7b5 github.com/couchbase/goxdcr/base.authClient+0x75 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:236 # 0x7e28ec github.com/couchbase/goxdcr/base.NewConn+0x1ec /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:612 # 0x7df454 github.com/couchbase/goxdcr/base.(*connPool).newConn+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:208 # 0x7df3da github.com/couchbase/goxdcr/base.(*connPool).GetNew+0x1a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:204 # 0x954be1 github.com/couchbase/goxdcr/parts.(*XmemNozzle).getClientWithRetry.func1+0x2a1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3774 # 0x8dbd49 github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithFinishSignal+0x109 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2615 # 0x954704 github.com/couchbase/goxdcr/parts.(*XmemNozzle).getClientWithRetry+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3777 # 0x94a38d github.com/couchbase/goxdcr/parts.(*XmemNozzle).initializeConnection+0x38d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2406 # 0x94b32b github.com/couchbase/goxdcr/parts.(*XmemNozzle).initialize+0x72b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2550 # 0x93bf0a github.com/couchbase/goxdcr/parts.(*XmemNozzle).Start+0x1ca /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:807 # 0x9c86a1 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart+0x321 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:233 # 0x9c8af9 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart.func1+0xb9 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:209 1 @ 0x43e876 0x437117 0x46b6c9 0x4dc712 0x4ddaf9 0x4ddae7 0x5d6b09 0x5e6c65 0x4ba65a 0x68b539 0x68b503 0x744545 0x74450d 0x73c4b3 0x742a93 0x8dfb23 0xd9d8e3 0xda1a71 0xd9ffa5 0x4711c1 # 0x46b6c8 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dc711 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4ddaf8 internal/poll.(*pollDesc).waitRead+0x298 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4ddae6 internal/poll.(*FD).Read+0x286 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:167 # 0x5d6b08 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e6c64 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x4ba659 io.ReadAtLeast+0x99 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:332 # 0x68b538 io.ReadFull+0xd8 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:351 # 0x68b502 github.com/couchbase/gomemcached.(*MCResponse).ReceiveWithBuf+0xa2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:169 # 0x744544 github.com/couchbase/gomemcached.(*MCResponse).Receive+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:156 # 0x74450c github.com/couchbase/gomemcached/client.getResponse+0x6c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/transport.go:35 # 0x73c4b2 github.com/couchbase/gomemcached/client.(*Client).Receive+0x52 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:333 # 0x742a92 github.com/couchbase/gomemcached/client.(*Client).GetAllVbSeqnos+0xb2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:1559 # 0x8dfb22 github.com/couchbase/goxdcr/utils.(*Utilities).GetHighSeqNos+0x2c2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:3031 # 0xd9d8e2 github.com/couchbase/goxdcr/service_impl.(*BucketTopologyService).getHighSeqnosUpdater.func1+0x942 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:814 # 0xda1a70 github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).updateOnce+0x570 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1362 # 0xd9ffa4 github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).run.func2+0x1e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1176 1 @ 0x43e876 0x437117 0x46b6c9 0x4dc712 0x4ddaf9 0x4ddae7 0x5d6b09 0x5e6c65 0x692afb 0x8c2b25 0x681ba8 0x68179b 0x6813d8 0x8c1dc5 0x8c1406 0x8c0f5b 0x8c0687 0x8c323b 0x8c3938 0x8c5c9d 0x8c5f85 0x4711c1 # 0x46b6c8 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dc711 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4ddaf8 internal/poll.(*pollDesc).waitRead+0x298 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4ddae6 internal/poll.(*FD).Read+0x286 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:167 # 0x5d6b08 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e6c64 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x692afa bufio.(*Reader).Read+0x1ba /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/bufio/bufio.go:237 # 0x8c2b24 github.com/couchbase/cbauth/revrpc.(*minirwc).Read+0x24 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/revrpc/revrpc.go:102 # 0x681ba7 encoding/json.(*Decoder).refill+0x187 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/encoding/json/stream.go:165 # 0x68179a encoding/json.(*Decoder).readValue+0xba /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/encoding/json/stream.go:140 # 0x6813d7 encoding/json.(*Decoder).Decode+0x77 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/encoding/json/stream.go:63 # 0x8c1dc4 net/rpc/jsonrpc.(*serverCodec).ReadRequestHeader+0x84 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/rpc/jsonrpc/server.go:66 # 0x8c1405 net/rpc.(*Server).readRequestHeader+0x65 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/rpc/server.go:588 # 0x8c0f5a net/rpc.(*Server).readRequest+0x3a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/rpc/server.go:548 # 0x8c0686 net/rpc.(*Server).ServeCodec+0x86 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/rpc/server.go:463 # 0x8c323a github.com/couchbase/cbauth/revrpc.(*Service).Run+0x5ba /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/revrpc/revrpc.go:192 # 0x8c3937 github.com/couchbase/cbauth/revrpc.BabysitService+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/revrpc/revrpc.go:288 # 0x8c5c9c github.com/couchbase/cbauth.runRPCForSvc+0xbc /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/default.go:57 # 0x8c5f84 github.com/couchbase/cbauth.startDefault.func1+0x24 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/default.go:66 1 @ 0x43e876 0x437117 0x46b6c9 0x4dc712 0x4ddaf9 0x4ddae7 0x5d6b09 0x5e6c65 0x7068ff 0x4711c1 # 0x46b6c8 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dc711 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4ddaf8 internal/poll.(*pollDesc).waitRead+0x298 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4ddae6 internal/poll.(*FD).Read+0x286 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:167 # 0x5d6b08 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e6c64 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x7068fe net/http.(*connReader).backgroundRead+0x3e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/server.go:674 1 @ 0x43e876 0x437117 0x46b6c9 0x4dc712 0x4e201d 0x4e200b 0x5d8c95 0x5efb25 0x5eec1d 0x711765 0x71139d 0x9b7f68 0x4711c1 # 0x46b6c8 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dc711 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4e201c internal/poll.(*pollDesc).waitRead+0x2bc /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4e200a internal/poll.(*FD).Accept+0x2aa /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:614 # 0x5d8c94 net.(*netFD).accept+0x34 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_unix.go:172 # 0x5efb24 net.(*TCPListener).accept+0x24 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/tcpsock_posix.go:148 # 0x5eec1c net.(*TCPListener).Accept+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/tcpsock.go:297 # 0x711764 net/http.(*Server).Serve+0x384 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/server.go:3059 # 0x71139c net/http.(*Server).ListenAndServe+0x7c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/server.go:2988 # 0x9b7f67 github.com/couchbase/goxdcr/adminport.(*httpServer).Start.func1+0xe7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/adminport/admin_httpd.go:93 1 @ 0x43e876 0x44ee3e 0x721106 0x71d01e 0x705219 0x6c5d37 0x6c55bb 0x6c78bb 0x8d87d3 0x8d87cb 0x8d7a53 0x8d0e5e 0x98b78a 0x4711c1 # 0x721105 net/http.(*Transport).getConn+0x5c5 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:1382 # 0x71d01d net/http.(*Transport).roundTrip+0x79d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:590 # 0x705218 net/http.(*Transport).RoundTrip+0x18 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/roundtrip.go:17 # 0x6c5d36 net/http.send+0x5f6 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:252 # 0x6c55ba net/http.(*Client).send+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:176 # 0x6c78ba net/http.(*Client).do+0x8fa /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:716 # 0x8d87d2 net/http.(*Client).Do+0x52 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:582 # 0x8d87ca github.com/couchbase/goxdcr/utils.(*Utilities).doRestCall+0x4a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2234 # 0x8d7a52 github.com/couchbase/goxdcr/utils.(*Utilities).QueryRestApiWithAuth+0x1b2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2096 # 0x8d0e5d github.com/couchbase/goxdcr/utils.(*Utilities).GetClusterInfoWStatusCode+0x1fd /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:1152 # 0x98b789 github.com/couchbase/goxdcr/metadata_svc.(*refreshContext).verifyNodeAndGetList.func1+0x569 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/remote_cluster_service.go:756 1 @ 0x43e876 0x44ee3e 0x729154 0x71d07a 0x705219 0x6c5d37 0x6c55bb 0x6c78bb 0x8d87d3 0x8d87cb 0x8d7a53 0x8f2002 0x904732 0x90470c 0x9069bf 0x4711c1 # 0x729153 net/http.(*persistConn).roundTrip+0x993 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:2638 # 0x71d079 net/http.(*Transport).roundTrip+0x7f9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:603 # 0x705218 net/http.(*Transport).RoundTrip+0x18 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/roundtrip.go:17 # 0x6c5d36 net/http.send+0x5f6 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:252 # 0x6c55ba net/http.(*Client).send+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:176 # 0x6c78ba net/http.(*Client).do+0x8fa /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:716 # 0x8d87d2 net/http.(*Client).Do+0x52 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:582 # 0x8d87ca github.com/couchbase/goxdcr/utils.(*Utilities).doRestCall+0x4a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2234 # 0x8d7a52 github.com/couchbase/goxdcr/utils.(*Utilities).QueryRestApiWithAuth+0x1b2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2096 # 0x8f2001 github.com/couchbase/goxdcr/peerToPeer.generateRequest.func1+0x321 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerMsgs.go:379 # 0x904731 github.com/couchbase/goxdcr/peerToPeer.(*RequestCommon).CallBack+0x24f1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerMsgs.go:120 # 0x90470b github.com/couchbase/goxdcr/peerToPeer.(*VBMasterCheckHandler).handleRequest+0x24cb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/vbMasterCheckHandler.go:241 # 0x9069be github.com/couchbase/goxdcr/peerToPeer.(*VBMasterCheckHandler).handler.func1+0x13e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/vbMasterCheckHandler.go:335 1 @ 0x43e876 0x44ee3e 0x7e8a5b 0x8dbe69 0x954705 0x952d8c 0x94ff17 0x939314 0x94fa89 0x4711c1 # 0x7e8a5a github.com/couchbase/goxdcr/base.WaitForTimeoutOrFinishSignal+0x9a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/simple_utils.go:612 # 0x8dbe68 github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithFinishSignal+0x228 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2621 # 0x954704 github.com/couchbase/goxdcr/parts.(*XmemNozzle).getClientWithRetry+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3777 # 0x952d8b github.com/couchbase/goxdcr/parts.(*XmemNozzle).repairConn+0x1cb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3550 # 0x94ff16 github.com/couchbase/goxdcr/parts.(*XmemNozzle).resendIfTimeout+0x376 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3036 # 0x939313 github.com/couchbase/goxdcr/parts.(*requestBuffer).modSlot+0x53 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:296 # 0x94fa88 github.com/couchbase/goxdcr/parts.(*XmemNozzle).checkAndRepairBufferMonitor+0x348 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2995 1 @ 0x43e876 0x44ee3e 0x879f05 0x4711c1 # 0x879f04 github.com/couchbase/cbauth/cbauthimpl.(*cfgChangeNotifier).loop+0x84 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/cbauthimpl/impl.go:309 1 @ 0x43e876 0x44ee3e 0x87a4a7 0x4711c1 # 0x87a4a6 github.com/couchbase/cbauth/cbauthimpl.(*tlsNotifier).loop+0x66 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/cbauthimpl/impl.go:389 1 @ 0x43e876 0x44ee3e 0x8e626f 0x4711c1 # 0x8e626e github.com/couchbase/goxdcr/peerToPeer.(*DiscoveryHandler).handler+0xae /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/discoveryHandler.go:55 1 @ 0x43e876 0x44ee3e 0x8fac70 0x4711c1 # 0x8fac6f github.com/couchbase/goxdcr/peerToPeer.(*PeriodicPushHandler).handler.func2+0x8f /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/periodicPushHandler.go:80 1 @ 0x43e876 0x44ee3e 0x8fadd9 0x4711c1 # 0x8fadd8 github.com/couchbase/goxdcr/peerToPeer.(*PeriodicPushHandler).handler.func1+0x98 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/periodicPushHandler.go:62 1 @ 0x43e876 0x44ee3e 0x8fe96b 0x4711c1 # 0x8fe96a github.com/couchbase/goxdcr/peerToPeer.(*ReplicaReplicatorImpl).run+0xca /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/replicaReplicator.go:229 1 @ 0x43e876 0x44ee3e 0x9067b0 0x4711c1 # 0x9067af github.com/couchbase/goxdcr/peerToPeer.(*VBMasterCheckHandler).handler.func2+0x8f /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/vbMasterCheckHandler.go:345 1 @ 0x43e876 0x44ee3e 0x9d399f 0x4711c1 # 0x9d399e github.com/couchbase/goxdcr/supervisor.(*GenericSupervisor).waitForResponse+0x21e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/supervisor/generic_supervisor.go:288 1 @ 0x43e876 0x44ee3e 0xa13425 0xa13435 0xa109c2 0xa1088f 0xa0fc33 0xa0f40f 0xa0e525 0xa0da45 0x4711c1 # 0xa13424 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).getNotification+0xc4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:1029 # 0xa13434 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).subscribeToBucketTopologySvc.func1.2+0xd4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:975 # 0xa109c1 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).calculateChangesLeftAndDocsProcessedMainPipeline+0x101 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:670 # 0xa1088e github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).calculateChangesLeftAndDocProcessed+0x10e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:654 # 0xa0fc32 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).processCalculatedStats+0x72 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:548 # 0xa0f40e github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).processRawStats+0x7ee /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:528 # 0xa0e524 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).updateStatsOnce+0x144 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:355 # 0xa0da44 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).updateStats+0x3c4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:303 1 @ 0x43e876 0x44ee3e 0xa3eda5 0x4711c1 # 0xa3eda4 github.com/couchbase/goxdcr/resource_manager.(*ResourceManager).collectCpuUsage+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/resource_manager/resource_manager.go:496 1 @ 0x43e876 0x44ee3e 0xa3f0a5 0x4711c1 # 0xa3f0a4 github.com/couchbase/goxdcr/resource_manager.(*ResourceManager).manageResources+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/resource_manager/resource_manager.go:514 1 @ 0x43e876 0x44ee3e 0xa3fb45 0x4711c1 # 0xa3fb44 github.com/couchbase/goxdcr/resource_manager.(*ResourceManager).logStats+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/resource_manager/resource_manager.go:599 1 @ 0x43e876 0x44ee3e 0xa5147d 0x4711c1 # 0xa5147c github.com/couchbase/goxdcr/backfill_manager.(*BackfillMgr).runRetryMonitor+0xdc /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/backfill_manager/backfill_manager.go:1651 1 @ 0x43e876 0x44ee3e 0xdc1885 0x4711c1 # 0xdc1884 github.com/couchbase/goxdcr/service_impl.(*ThroughputThrottler).update+0x244 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/throughput_throttler_service.go:101 1 @ 0x43e876 0x44ee3e 0xdc1ea6 0x4711c1 # 0xdc1ea5 github.com/couchbase/goxdcr/service_impl.(*ThroughputThrottler).clearReassignedTokens+0x245 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/throughput_throttler_service.go:187 1 @ 0x43e876 0x44ee3e 0xdc2b3c 0x4711c1 # 0xdc2b3b github.com/couchbase/goxdcr/service_impl.(*ThroughputThrottler).logStats+0x1fb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/throughput_throttler_service.go:360 1 @ 0x43e876 0x44ee3e 0xdd8f76 0x4711c1 # 0xdd8f75 github.com/couchbase/goxdcr/replication_manager.(*Adminport).Start+0x3d5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/replication_manager/adminport.go:121 1 @ 0x43e876 0x44ee3e 0xdf63ca 0x4711c1 # 0xdf63c9 github.com/couchbase/goxdcr/replication_manager.(*replicationManager).checkReplicationStatus+0x1a9 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/replication_manager/replication_manager.go:445 1 @ 0x43e876 0x44ee3e 0xdfb725 0x4711c1 # 0xdfb724 github.com/couchbase/goxdcr/replication_manager.logMemStats+0x1c4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/replication_manager/replication_manager.go:1068 1 @ 0x43e876 0x44ff2f 0x44ff06 0x46d006 0x48c345 0xd9d61b 0xd9d604 0xda1a71 0xd9ffa5 0x4711c1 # 0x46d005 sync.runtime_SemacquireMutex+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:77 # 0x48c344 sync.(*Mutex).lockSlow+0x164 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:171 # 0xd9d61a sync.(*Mutex).Lock+0x67a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:90 # 0xd9d603 github.com/couchbase/goxdcr/service_impl.(*BucketTopologyService).getHighSeqnosUpdater.func1+0x663 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:799 # 0xda1a70 github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).updateOnce+0x570 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1362 # 0xd9ffa4 github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).run.func2+0x1e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1176 1 @ 0x43e876 0x46d2ac 0x46d28c 0x48a4ec 0x98a7b1 0x98c34c 0x990e5f 0x4711c1 # 0x46d28b sync.runtime_notifyListWait+0x12b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:527 # 0x48a4eb sync.(*Cond).Wait+0x8b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/cond.go:70 # 0x98a7b0 github.com/couchbase/goxdcr/metadata_svc.(*refreshContext).verifyNodeAndGetList+0x7d0 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/remote_cluster_service.go:782 # 0x98c34b github.com/couchbase/goxdcr/metadata_svc.(*RemoteClusterAgent).Refresh+0x3ab /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/remote_cluster_service.go:919 # 0x990e5e github.com/couchbase/goxdcr/metadata_svc.(*RemoteClusterAgent).runPeriodicRefresh+0x19e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/remote_cluster_service.go:1619 1 @ 0x43e876 0x46dfb5 0xd835a5 0x4711c1 # 0x46dfb4 time.Sleep+0x134 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/time.go:195 # 0xd835a4 github.com/couchbase/eventing-ee/evaluator/store.(*storage).periodicScan+0x44 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing-ee/evaluator/store/store.go:328 1 @ 0x4c2767 0x4c0865 0x4ddb2e 0x4ddb16 0x4dd896 0x4e807e 0x4e8076 0x69251f 0x692cec 0xdfb3ca 0x4711c1 # 0x4c2766 syscall.Syscall+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/syscall/syscall_linux.go:69 # 0x4c0864 syscall.read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/syscall/zsyscall_linux_amd64.go:711 # 0x4ddb2d syscall.Read+0x2cd /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/syscall/syscall_unix.go:178 # 0x4ddb15 internal/poll.ignoringEINTRIO+0x2b5 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:794 # 0x4dd895 internal/poll.(*FD).Read+0x35 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:163 # 0x4e807d os.(*File).read+0x5d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file_posix.go:31 # 0x4e8075 os.(*File).Read+0x55 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file.go:118 # 0x69251e bufio.(*Reader).fill+0xfe /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/bufio/bufio.go:106 # 0x692ceb bufio.(*Reader).ReadByte+0x2b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/bufio/bufio.go:265 # 0xdfb3c9 github.com/couchbase/goxdcr/replication_manager.pollStdin+0x149 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/replication_manager/replication_manager.go:1039