goroutine profile: total 6369 1734 @ 0x43f4b6 0x44fa7e 0x90e833 0x471e01 # 0x90e832 github.com/couchbase/goxdcr/component.(*AsyncComponentEventListenerImpl).run+0x92 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/component/async_listener.go:68 295 @ 0x471e01 242 @ 0x43f4b6 0x44fa7e 0x96bddd 0x471e01 # 0x96bddc github.com/couchbase/goxdcr/metadata_svc.(*CheckpointsServiceCacheImpl).Run+0x17c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/checkpointsServiceCache.go:263 174 @ 0x43f4b6 0x44fa7e 0x728ef2 0x471e01 # 0x728ef1 net/http.(*persistConn).writeLoop+0xf1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:2410 165 @ 0x43f4b6 0x437d57 0x46c309 0x4dd352 0x4de739 0x4de727 0x5d7749 0x5e78a5 0x72640e 0x69315f 0x6932bd 0x72722c 0x471e01 # 0x46c308 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dd351 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4de738 internal/poll.(*pollDesc).waitRead+0x298 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4de726 internal/poll.(*FD).Read+0x286 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:167 # 0x5d7748 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e78a4 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x72640d net/http.(*persistConn).Read+0x4d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:1943 # 0x69315e bufio.(*Reader).fill+0xfe /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/bufio/bufio.go:106 # 0x6932bc bufio.(*Reader).Peek+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/bufio/bufio.go:144 # 0x72722b net/http.(*persistConn).readLoop+0x1ab /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:2107 125 @ 0x43f4b6 0x44fa7e 0x729d94 0x71dcba 0x705e59 0x6c6977 0x6c61fb 0x6c84fb 0x8d9413 0x8d940b 0x8da07e 0xda8dc9 0xda75e5 0x9eb9bd 0x9ea94c 0x471e01 # 0x729d93 net/http.(*persistConn).roundTrip+0x993 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:2638 # 0x71dcb9 net/http.(*Transport).roundTrip+0x7f9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:603 # 0x705e58 net/http.(*Transport).RoundTrip+0x18 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/roundtrip.go:17 # 0x6c6976 net/http.send+0x5f6 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:252 # 0x6c61fa net/http.(*Client).send+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:176 # 0x6c84fa net/http.(*Client).do+0x8fa /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:716 # 0x8d9412 net/http.(*Client).Do+0x52 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:582 # 0x8d940a github.com/couchbase/goxdcr/utils.(*Utilities).doRestCall+0x4a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2234 # 0x8da07d github.com/couchbase/goxdcr/utils.(*Utilities).InvokeRestWithRetryWithAuth+0x37d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2321 # 0xda8dc8 github.com/couchbase/goxdcr/service_impl.(*CAPIService).send_post+0x308 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/capi_service.go:291 # 0xda75e4 github.com/couchbase/goxdcr/service_impl.(*CAPIService).PreReplicate+0x144 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/capi_service.go:122 # 0x9eb9bc github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).getDataFromCkpts+0x4bc /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1444 # 0x9ea94b github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).startSeqnoGetter+0x2cb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1321 124 @ 0x43f4b6 0x450b6f 0x450b46 0x46dc46 0x48cf85 0x50ecbc 0x50ec9b 0x50f0dd 0x5121fc 0x9ea2e5 0x9ea193 0x9ea9d4 0x471e01 # 0x46dc45 sync.runtime_SemacquireMutex+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:77 # 0x48cf84 sync.(*Mutex).lockSlow+0x164 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:171 # 0x50ecbb sync.(*Mutex).Lock+0x9b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:90 # 0x50ec9a log.(*Logger).Output+0x7a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:175 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0x9ea2e4 github.com/couchbase/goxdcr/log.(*CommonLogger).Infof+0x204 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:193 # 0x9ea192 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).setTimestampForVB+0xb2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1285 # 0x9ea9d3 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).startSeqnoGetter+0x353 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1335 122 @ 0x43f4b6 0x450b6f 0x450b46 0x46db27 0x48ea8b 0x9e8816 0xdd18e5 0x471e01 # 0x46db26 sync.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:62 # 0x48ea8a sync.(*WaitGroup).Wait+0x4a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/waitgroup.go:116 # 0x9e8815 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).SetVBTimestamps+0xe55 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1164 # 0xdd18e4 github.com/couchbase/goxdcr/factory.(*XDCRFactory).SetStartSeqno+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/factory/xdcr_factory.go:888 121 @ 0x43f4b6 0x44fa7e 0x97909f 0x471e01 # 0x97909e github.com/couchbase/goxdcr/metadata_svc.(*CollectionsManifestAgent).runPersistRequestHandler+0xbe /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/collections_manifest_service.go:528 120 @ 0x43f4b6 0x44fa7e 0x901138 0x471e01 # 0x901137 github.com/couchbase/goxdcr/peerToPeer.(*ReplicatorAgentImpl).run+0x457 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/replicaReplicator.go:508 120 @ 0x43f4b6 0x44fa7e 0xa56545 0x471e01 # 0xa56544 github.com/couchbase/goxdcr/backfill_manager.(*BackfillRequestHandler).run+0x2a4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/backfill_manager/backfill_request_handler.go:241 119 @ 0x43f4b6 0x44fa7e 0x8fdd7d 0x471e01 # 0x8fdd7c github.com/couchbase/goxdcr/peerToPeer.(*ReplicaCacheMonitor).Run+0x9c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/replicaCache.go:108 115 @ 0x43f4b6 0x44fa7e 0x94f57b 0x471e01 # 0x94f57a github.com/couchbase/goxdcr/parts.(*XmemNozzle).selfMonitor+0x1da /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2912 113 @ 0x43f4b6 0x44fa7e 0x747599 0x471e01 # 0x747598 github.com/couchbase/gomemcached/client.(*UprFeed).sendCommands+0xb8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/upr_feed.go:346 101 @ 0x43f4b6 0x44fa7e 0x95049a 0x471e01 # 0x950499 github.com/couchbase/goxdcr/parts.(*XmemNozzle).checkAndRepairBufferMonitor+0x119 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2985 101 @ 0x43f4b6 0x44fa7e 0x978f1e 0x471e01 # 0x978f1d github.com/couchbase/goxdcr/metadata_svc.(*CollectionsManifestAgent).runPeriodicRefresh+0xdd /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/collections_manifest_service.go:504 98 @ 0x43f4b6 0x44fa7e 0x9288b1 0x471e01 # 0x9288b0 github.com/couchbase/goxdcr/parts.(*DcpNozzle).collectDcpDataChanLen+0x110 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1948 96 @ 0x43f4b6 0x44fa7e 0x927a85 0x471e01 # 0x927a84 github.com/couchbase/goxdcr/parts.(*DcpNozzle).checkInactiveUprStreams+0x124 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1791 83 @ 0x43f4b6 0x44fa7e 0x939a14 0x93f43a 0x93f025 0x471e01 # 0x939a13 github.com/couchbase/goxdcr/parts.(*requestBuffer).flowControl+0x73 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:237 # 0x93f439 github.com/couchbase/goxdcr/parts.(*XmemNozzle).processBatch+0x39 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1120 # 0x93f024 github.com/couchbase/goxdcr/parts.(*XmemNozzle).processData_sendbatch+0x4e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1077 81 @ 0x43f4b6 0x450b6f 0x450b46 0x46db87 0x4dcdd2 0x4e06f2 0x4e0703 0x4e9465 0x4e945b 0x5118e6 0x50ef6a 0x50f0dd 0x5121fc 0x9ea2e5 0x9ea193 0x9ea9d4 0x471e01 # 0x46db86 internal/poll.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:67 # 0x4dcdd1 internal/poll.(*fdMutex).rwlock+0xd1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:154 # 0x4e06f1 internal/poll.(*FD).writeLock+0x71 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:239 # 0x4e0702 internal/poll.(*FD).Write+0x82 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:370 # 0x4e9464 os.(*File).write+0x64 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file_posix.go:48 # 0x4e945a os.(*File).Write+0x5a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file.go:175 # 0x5118e5 github.com/couchbase/goxdcr/log.(*LogWriter).Write+0x25 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:94 # 0x50ef69 log.(*Logger).Output+0x349 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:194 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0x9ea2e4 github.com/couchbase/goxdcr/log.(*CommonLogger).Infof+0x204 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:193 # 0x9ea192 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).setTimestampForVB+0xb2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1285 # 0x9ea9d3 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).startSeqnoGetter+0x353 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1335 80 @ 0x43f4b6 0x44fa7e 0x9247a5 0x471e01 # 0x9247a4 github.com/couchbase/goxdcr/parts.(*DcpNozzle).startUprStreams+0x224 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1344 75 @ 0x43f4b6 0x44fa7e 0x9ba50a 0x471e01 # 0x9ba509 github.com/couchbase/goxdcr/gen_server.(*GenServer).run+0xc9 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/gen_server/gen_server.go:83 74 @ 0x43f4b6 0x44fa7e 0x9d3985 0x471e01 # 0x9d3984 github.com/couchbase/goxdcr/supervisor.(*GenericSupervisor).supervising+0x184 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/supervisor/generic_supervisor.go:207 74 @ 0x43f4b6 0x44fa7e 0x9de138 0x471e01 # 0x9de137 github.com/couchbase/goxdcr/pipeline_svc.(*BandwidthThrottler).update+0x237 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/bandwidth_throttler.go:135 74 @ 0x43f4b6 0x44fa7e 0xa0b4e5 0x471e01 # 0xa0b4e4 github.com/couchbase/goxdcr/pipeline_svc.(*PipelineSupervisor).monitorPipelineHealth.func1+0x124 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/pipeline_supervisor.go:196 74 @ 0x43f4b6 0x44fa7e 0xa0be88 0x471e01 # 0xa0be87 github.com/couchbase/goxdcr/pipeline_svc.(*PipelineSupervisor).checkAndLogFilterErrors+0x1a7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/pipeline_supervisor.go:273 74 @ 0x43f4b6 0x44fa7e 0xa137d9 0x471e01 # 0xa137d8 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).subscribeToBucketTopologySvc+0x158 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:954 73 @ 0x43f4b6 0x44fa7e 0x9ede70 0x471e01 # 0x9ede6f github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).checkpointing+0x40f /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1659 72 @ 0x43f4b6 0x44fa7e 0xa14585 0x471e01 # 0xa14584 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).watchNotificationCh+0x144 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:1041 66 @ 0x43f4b6 0x44fa7e 0xa24f97 0x471e01 # 0xa24f96 github.com/couchbase/goxdcr/pipeline_svc.(*TopologyChangeDetectorSvc).monitorSource.func1+0xd6 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/topology_change_detector.go:483 65 @ 0x43f4b6 0x44fa7e 0xa0ee25 0x471e01 # 0xa0ee24 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).logStats+0x224 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:332 64 @ 0x43f4b6 0x44fa7e 0xa015e9 0x471e01 # 0xa015e8 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).periodicMergerImpl+0xa8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:3518 63 @ 0x43f4b6 0x44fa7e 0xa25b34 0x471e01 # 0xa25b33 github.com/couchbase/goxdcr/pipeline_svc.(*TopologyChangeDetectorSvc).monitorTarget.func1+0xd3 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/topology_change_detector.go:572 59 @ 0x43f4b6 0x437d57 0x46c309 0x4dd352 0x4de739 0x4de727 0x5d7749 0x5e78a5 0x4bb29a 0x68c179 0x68c143 0x745185 0x74514d 0x73d0f3 0x9534ab 0x94c9c5 0x471e01 # 0x46c308 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dd351 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4de738 internal/poll.(*pollDesc).waitRead+0x298 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4de726 internal/poll.(*FD).Read+0x286 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:167 # 0x5d7748 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e78a4 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x4bb299 io.ReadAtLeast+0x99 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:332 # 0x68c178 io.ReadFull+0xd8 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:351 # 0x68c142 github.com/couchbase/gomemcached.(*MCResponse).ReceiveWithBuf+0xa2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:169 # 0x745184 github.com/couchbase/gomemcached.(*MCResponse).Receive+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:156 # 0x74514c github.com/couchbase/gomemcached/client.getResponse+0x6c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/transport.go:35 # 0x73d0f2 github.com/couchbase/gomemcached/client.(*Client).Receive+0x52 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:333 # 0x9534aa github.com/couchbase/goxdcr/parts.(*XmemNozzle).readFromClient+0x18a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3498 # 0x94c9c4 github.com/couchbase/goxdcr/parts.(*XmemNozzle).receiveResponse+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2618 53 @ 0x43f4b6 0x44fa7e 0x95484b 0x93e5e6 0x93dbb6 0x90f615 0x921da2 0x471e01 # 0x95484a github.com/couchbase/goxdcr/parts.(*XmemNozzle).writeToDataChan+0xea /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3664 # 0x93e5e5 github.com/couchbase/goxdcr/parts.(*XmemNozzle).accumuBatch+0x3e5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:955 # 0x93dbb5 github.com/couchbase/goxdcr/parts.(*XmemNozzle).Receive+0x115 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:907 # 0x90f614 github.com/couchbase/goxdcr/connector.(*Router).Forward+0x294 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/connector/router.go:118 # 0x921da1 github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0xcc1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1102 52 @ 0x43f4b6 0x437d57 0x46c309 0x4dd352 0x4de739 0x4de727 0x5d7749 0x5e78a5 0x4bb29a 0x68b4dc 0x68b4ae 0x74ab8d 0x471e01 # 0x46c308 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dd351 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4de738 internal/poll.(*pollDesc).waitRead+0x298 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4de726 internal/poll.(*FD).Read+0x286 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:167 # 0x5d7748 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e78a4 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x4bb299 io.ReadAtLeast+0x99 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:332 # 0x68b4db io.ReadFull+0x9b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:351 # 0x68b4ad github.com/couchbase/gomemcached.(*MCRequest).Receive+0x6d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_req.go:643 # 0x74ab8c github.com/couchbase/gomemcached/client.(*UprFeed).runFeed+0x14c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/upr_feed.go:904 52 @ 0x43f4b6 0x44fa7e 0xa3761c 0x48d322 0xa373c7 0xa37395 0x471e01 # 0xa3761b github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run.func1+0x21b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1599 # 0x48d321 sync.(*Once).doSlow+0xc1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:74 # 0xa373c6 sync.(*Once).Do+0x46 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:65 # 0xa37394 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run+0x14 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1588 46 @ 0x43f4b6 0x44fa7e 0x9ffe1f 0x9f9015 0xa352f0 0x8fd4f9 0x8fcc6a 0x8fc565 0x471e01 # 0x9ffe1e github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).mergePeerNodesPeriodicPush+0x7de /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:3412 # 0x9f9014 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).MergePeerNodesCkptInfo+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:2580 # 0xa352ef github.com/couchbase/goxdcr/pipeline_manager.(*PipelineManager).HandlePeerCkptPush+0x24f /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1272 # 0x8fd4f8 github.com/couchbase/goxdcr/peerToPeer.(*PeriodicPushHandler).storePushReqInfoByType+0xf8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/periodicPushHandler.go:229 # 0x8fcc69 github.com/couchbase/goxdcr/peerToPeer.(*PeriodicPushHandler).storePushRequestInfo+0x349 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/periodicPushHandler.go:190 # 0x8fc564 github.com/couchbase/goxdcr/peerToPeer.(*PeriodicPushHandler).handleRequest.func1+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/periodicPushHandler.go:120 38 @ 0x43f4b6 0x450b6f 0x450b46 0x46db27 0x48ea8b 0x9c8887 0x7e7482 0x471e01 # 0x46db26 sync.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:62 # 0x48ea8a sync.(*WaitGroup).Wait+0x4a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/waitgroup.go:116 # 0x9c8886 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPartsWithTimeout.func1+0x226 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:161 # 0x7e7481 github.com/couchbase/goxdcr/base.ExecWithTimeout.func1+0x21 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/simple_utils.go:48 37 @ 0x43f4b6 0x44fa7e 0xda0c50 0x471e01 # 0xda0c4f github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).run.func2+0x18f /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1169 35 @ 0x43f4b6 0x44fa7e 0x5d9666 0x471e01 # 0x5d9665 net.(*netFD).connect.func2+0x85 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_unix.go:118 34 @ 0x43f4b6 0x44fa7e 0x94c945 0x471e01 # 0x94c944 github.com/couchbase/goxdcr/parts.(*XmemNozzle).receiveResponse+0xe4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2608 33 @ 0x43f4b6 0x40b88e 0x40b43d 0x9b98f2 0x7e7a36 0x471e01 # 0x9b98f1 github.com/couchbase/goxdcr/adminport.(*httpServer).systemHandler.func2+0xb1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/adminport/admin_httpd.go:136 # 0x7e7a35 github.com/couchbase/goxdcr/base.ExecWithTimeout3.func1+0x35 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/simple_utils.go:115 32 @ 0x43f4b6 0x44fa7e 0xa14125 0xa14135 0xa116c2 0xa1158f 0xa10933 0xa1010f 0xa0f225 0xa0e765 0x471e01 # 0xa14124 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).getNotification+0xc4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:1029 # 0xa14134 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).subscribeToBucketTopologySvc.func1.2+0xd4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:975 # 0xa116c1 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).calculateChangesLeftAndDocsProcessedMainPipeline+0x101 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:670 # 0xa1158e github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).calculateChangesLeftAndDocProcessed+0x10e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:654 # 0xa10932 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).processCalculatedStats+0x72 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:548 # 0xa1010e github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).processRawStats+0x7ee /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:528 # 0xa0f224 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).updateStatsOnce+0x144 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:355 # 0xa0e764 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).updateStats+0x3e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:309 31 @ 0x43f4b6 0x437d57 0x46c309 0x4dd352 0x5d9179 0x5d915d 0x5d9181 0x5ecb05 0x5ec632 0x5e3b98 0x5f02cf 0x5f0169 0x5ce992 0x5ce2e5 0x5cd953 0x5cd36c 0x721087 0x723f8c 0x7229d0 0x471e01 # 0x46c308 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dd351 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x5d9178 internal/poll.(*pollDesc).waitWrite+0x798 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:93 # 0x5d915c internal/poll.(*FD).WaitWrite+0x77c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:741 # 0x5d9180 net.(*netFD).connect+0x7a0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_unix.go:141 # 0x5ecb04 net.(*netFD).dial+0x3c4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/sock_posix.go:151 # 0x5ec631 net.socket+0x2d1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/sock_posix.go:70 # 0x5e3b97 net.internetSocket+0xf7 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/ipsock_posix.go:142 # 0x5f02ce net.(*sysDialer).doDialTCP+0xee /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/tcpsock_posix.go:74 # 0x5f0168 net.(*sysDialer).dialTCP+0x68 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/tcpsock_posix.go:64 # 0x5ce991 net.(*sysDialer).dialSingle+0x1f1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:580 # 0x5ce2e4 net.(*sysDialer).dialSerial+0x244 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:548 # 0x5cd952 net.(*sysDialer).dialParallel+0x412 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:449 # 0x5cd36b net.(*Dialer).DialContext+0x70b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:440 # 0x721086 net/http.(*Transport).dial+0xe6 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:1176 # 0x723f8b net/http.(*Transport).dialConn+0x82b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:1614 # 0x7229cf net/http.(*Transport).dialConnFor+0xaf /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:1456 31 @ 0x43f4b6 0x44fa7e 0x7e775c 0x9b90e5 0x9b9c9d 0x710689 0x711e36 0x70d812 0x471e01 # 0x7e775b github.com/couchbase/goxdcr/base.ExecWithTimeout3+0x29b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/simple_utils.go:124 # 0x9b90e4 github.com/couchbase/goxdcr/adminport.(*httpServer).systemHandler+0x1c4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/adminport/admin_httpd.go:131 # 0x9b9c9c github.com/couchbase/goxdcr/adminport.(*Handler).ServeHTTP+0x13c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/adminport/admin_httpd.go:204 # 0x710688 net/http.(*ServeMux).ServeHTTP+0x148 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/server.go:2500 # 0x711e35 net/http.serverHandler.ServeHTTP+0x315 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/server.go:2936 # 0x70d811 net/http.(*conn).serve+0x611 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/server.go:1995 31 @ 0x43f4b6 0x450b6f 0x450b46 0x46dc46 0x48cf85 0x48e5d6 0x48e5b5 0x68e6ce 0x74715e 0x74afbb 0x471e01 # 0x46dc45 sync.runtime_SemacquireMutex+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:77 # 0x48cf84 sync.(*Mutex).lockSlow+0x164 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:171 # 0x48e5d5 sync.(*Mutex).Lock+0x35 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:90 # 0x48e5b4 sync.(*RWMutex).Lock+0x14 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/rwmutex.go:147 # 0x68e6cd github.com/couchbase/goutils/logging.Infof+0x6d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goutils/logging/logger.go:289 # 0x74715d github.com/couchbase/gomemcached/client.(*vbStreamNegotiator).handleStreamRequest+0x3fd /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/upr_feed.go:242 # 0x74afba github.com/couchbase/gomemcached/client.(*UprFeed).runFeed+0x57a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/upr_feed.go:933 24 @ 0x43f4b6 0x44fa7e 0x93ece5 0x471e01 # 0x93ece4 github.com/couchbase/goxdcr/parts.(*XmemNozzle).processData_sendbatch+0x1a4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1036 23 @ 0x43f4b6 0x450b6f 0x450b46 0x46dc46 0x48cf85 0x50ecbc 0x50ec9b 0x50f0dd 0x5121fc 0x8ed5b0 0x8ed54f 0x471e01 # 0x46dc45 sync.runtime_SemacquireMutex+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:77 # 0x48cf84 sync.(*Mutex).lockSlow+0x164 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:171 # 0x50ecbb sync.(*Mutex).Lock+0x9b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:90 # 0x50ec9a log.(*Logger).Output+0x7a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:175 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0x8ed5af github.com/couchbase/goxdcr/log.(*CommonLogger).Errorf+0x38f /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:185 # 0x8ed54e github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).sendToSpecifiedPeersOnce.func1.1+0x32e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:311 22 @ 0x43f4b6 0x40c69d 0x40c198 0xda0951 0x471e01 # 0xda0950 github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).run+0x350 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1186 22 @ 0x43f4b6 0x450b6f 0x450b46 0x46db87 0x4dcdd2 0x4e06f2 0x4e0703 0x4e9465 0x4e945b 0x5118e6 0x50ef6a 0x50f0dd 0x5121fc 0xa11c73 0xa11b9e 0xa117ea 0xa1158f 0xa10933 0xa1010f 0xa0f225 0xa0e765 0x471e01 # 0x46db86 internal/poll.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:67 # 0x4dcdd1 internal/poll.(*fdMutex).rwlock+0xd1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:154 # 0x4e06f1 internal/poll.(*FD).writeLock+0x71 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:239 # 0x4e0702 internal/poll.(*FD).Write+0x82 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:370 # 0x4e9464 os.(*File).write+0x64 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file_posix.go:48 # 0x4e945a os.(*File).Write+0x5a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file.go:175 # 0x5118e5 github.com/couchbase/goxdcr/log.(*LogWriter).Write+0x25 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:94 # 0x50ef69 log.(*Logger).Output+0x349 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:194 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0xa11c72 github.com/couchbase/goxdcr/log.(*CommonLogger).Infof+0x112 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:193 # 0xa11b9d github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).logChangesLeft+0x3d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:699 # 0xa117e9 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).calculateChangesLeftAndDocsProcessedMainPipeline+0x229 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:684 # 0xa1158e github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).calculateChangesLeftAndDocProcessed+0x10e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:654 # 0xa10932 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).processCalculatedStats+0x72 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:548 # 0xa1010e github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).processRawStats+0x7ee /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:528 # 0xa0f224 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).updateStatsOnce+0x144 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:355 # 0xa0e764 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).updateStats+0x3e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:309 21 @ 0x43f4b6 0x450b6f 0x450b46 0x46db27 0x48ea8b 0x9c9594 0x9c8d5f 0x471e01 # 0x46db26 sync.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:62 # 0x48ea8a sync.(*WaitGroup).Wait+0x4a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/waitgroup.go:116 # 0x9c9593 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart+0x5d3 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:214 # 0x9c8d5e github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPartsWithTimeout.func1.1+0xde /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:153 19 @ 0x43f4b6 0x46ebf5 0x8dc7c8 0x8dc47b 0x8ec569 0x8ec1dd 0x8eee2d 0xdd1e51 0x9cb509 0x9c9f8d 0xa2f318 0xa39525 0xa379c9 0x48d322 0xa373c7 0xa37395 0x471e01 # 0x46ebf4 time.Sleep+0x134 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/time.go:195 # 0x8dc7c7 github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithOriginalError+0x1c7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2593 # 0x8dc47a github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutor+0x3a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2575 # 0x8ec568 github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).sendToSpecifiedPeersOnce+0x328 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:356 # 0x8ec1dc github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).sendToEachPeerOnce+0xfc /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:257 # 0x8eee2c github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).CheckVBMaster+0x30c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:541 # 0xdd1e50 github.com/couchbase/goxdcr/factory.(*XDCRFactory).PreReplicationVBMasterCheck+0x2b0 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/factory/xdcr_factory.go:933 # 0x9cb508 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).runP2PProtocol+0xe8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:376 # 0x9c9f8c github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).Start+0x76c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:274 # 0xa2f317 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineManager).StartPipeline+0x4d7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:464 # 0xa39524 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).update+0x424 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1899 # 0xa379c8 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run.func1+0x5c8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1610 # 0x48d321 sync.(*Once).doSlow+0xc1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:74 # 0xa373c6 sync.(*Once).Do+0x46 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:65 # 0xa37394 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run+0x14 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1588 18 @ 0x43f4b6 0x44fa7e 0x721d46 0x71dc5e 0x705e59 0x6c6977 0x6c61fb 0x6c84fb 0x8d9413 0x8d940b 0x8da07e 0xda8dc9 0xda75e5 0x9eb9bd 0x9ea94c 0x471e01 # 0x721d45 net/http.(*Transport).getConn+0x5c5 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:1382 # 0x71dc5d net/http.(*Transport).roundTrip+0x79d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:590 # 0x705e58 net/http.(*Transport).RoundTrip+0x18 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/roundtrip.go:17 # 0x6c6976 net/http.send+0x5f6 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:252 # 0x6c61fa net/http.(*Client).send+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:176 # 0x6c84fa net/http.(*Client).do+0x8fa /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:716 # 0x8d9412 net/http.(*Client).Do+0x52 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:582 # 0x8d940a github.com/couchbase/goxdcr/utils.(*Utilities).doRestCall+0x4a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2234 # 0x8da07d github.com/couchbase/goxdcr/utils.(*Utilities).InvokeRestWithRetryWithAuth+0x37d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2321 # 0xda8dc8 github.com/couchbase/goxdcr/service_impl.(*CAPIService).send_post+0x308 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/capi_service.go:291 # 0xda75e4 github.com/couchbase/goxdcr/service_impl.(*CAPIService).PreReplicate+0x144 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/capi_service.go:122 # 0x9eb9bc github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).getDataFromCkpts+0x4bc /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1444 # 0x9ea94b github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).startSeqnoGetter+0x2cb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1321 17 @ 0x43f4b6 0x44fa7e 0x729d94 0x71dcba 0x705e59 0x6c6977 0x6c61fb 0x6c84fb 0x8d9413 0x8d940b 0x8d8693 0x8e9c62 0x8ed537 0x471e01 # 0x729d93 net/http.(*persistConn).roundTrip+0x993 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:2638 # 0x71dcb9 net/http.(*Transport).roundTrip+0x7f9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:603 # 0x705e58 net/http.(*Transport).RoundTrip+0x18 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/roundtrip.go:17 # 0x6c6976 net/http.send+0x5f6 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:252 # 0x6c61fa net/http.(*Client).send+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:176 # 0x6c84fa net/http.(*Client).do+0x8fa /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:716 # 0x8d9412 net/http.(*Client).Do+0x52 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:582 # 0x8d940a github.com/couchbase/goxdcr/utils.(*Utilities).doRestCall+0x4a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2234 # 0x8d8692 github.com/couchbase/goxdcr/utils.(*Utilities).QueryRestApiWithAuth+0x1b2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2096 # 0x8e9c61 github.com/couchbase/goxdcr/peerToPeer.(*P2pCommAPIimpl).P2PSend+0x241 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerCommAPI.go:133 # 0x8ed536 github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).sendToSpecifiedPeersOnce.func1.1+0x316 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:309 14 @ 0x43f4b6 0x450b6f 0x450b46 0x46db27 0x48ea8b 0x8ed046 0x8dc6bb 0x8dc47b 0x8ec569 0x8ec1dd 0x8eee2d 0xdd1e51 0x9cb509 0x9c9f8d 0xa2f318 0xa39525 0xa389fb 0xa37bc6 0x48d322 0xa373c7 0xa37395 0x471e01 # 0x46db26 sync.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:62 # 0x48ea8a sync.(*WaitGroup).Wait+0x4a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/waitgroup.go:116 # 0x8ed045 github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).sendToSpecifiedPeersOnce.func1+0x3e5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:344 # 0x8dc6ba github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithOriginalError+0xba /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2587 # 0x8dc47a github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutor+0x3a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2575 # 0x8ec568 github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).sendToSpecifiedPeersOnce+0x328 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:356 # 0x8ec1dc github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).sendToEachPeerOnce+0xfc /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:257 # 0x8eee2c github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).CheckVBMaster+0x30c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:541 # 0xdd1e50 github.com/couchbase/goxdcr/factory.(*XDCRFactory).PreReplicationVBMasterCheck+0x2b0 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/factory/xdcr_factory.go:933 # 0x9cb508 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).runP2PProtocol+0xe8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:376 # 0x9c9f8c github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).Start+0x76c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:274 # 0xa2f317 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineManager).StartPipeline+0x4d7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:464 # 0xa39524 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).update+0x424 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1899 # 0xa389fa github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).checkIfNeedToReUpdate+0x11a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1739 # 0xa37bc5 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run.func1+0x7c5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1621 # 0x48d321 sync.(*Once).doSlow+0xc1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:74 # 0xa373c6 sync.(*Once).Do+0x46 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:65 # 0xa37394 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run+0x14 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1588 14 @ 0x43f4b6 0x450b6f 0x450b46 0x46dc46 0x48cf85 0x50ecbc 0x50ec9b 0x50f0dd 0x5121fc 0xa3b985 0xa3b95e 0x471e01 # 0x46dc45 sync.runtime_SemacquireMutex+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:77 # 0x48cf84 sync.(*Mutex).lockSlow+0x164 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:171 # 0x50ecbb sync.(*Mutex).Lock+0x9b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:90 # 0x50ec9a log.(*Logger).Output+0x7a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:175 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0xa3b984 github.com/couchbase/goxdcr/log.(*CommonLogger).Infof+0x44 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:193 # 0xa3b95d github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).scheduleFutureRefresh.func1+0x1d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:2215 12 @ 0x43f4b6 0x450b6f 0x450b46 0x46db27 0x48ea8b 0x8ed046 0x8dc6bb 0x8dc47b 0x8ec569 0x8ec1dd 0x8eee2d 0xdd1e51 0x9cb509 0x9c9f8d 0xa2f318 0xa39525 0xa379c9 0x48d322 0xa373c7 0xa37395 0x471e01 # 0x46db26 sync.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:62 # 0x48ea8a sync.(*WaitGroup).Wait+0x4a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/waitgroup.go:116 # 0x8ed045 github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).sendToSpecifiedPeersOnce.func1+0x3e5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:344 # 0x8dc6ba github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithOriginalError+0xba /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2587 # 0x8dc47a github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutor+0x3a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2575 # 0x8ec568 github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).sendToSpecifiedPeersOnce+0x328 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:356 # 0x8ec1dc github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).sendToEachPeerOnce+0xfc /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:257 # 0x8eee2c github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).CheckVBMaster+0x30c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:541 # 0xdd1e50 github.com/couchbase/goxdcr/factory.(*XDCRFactory).PreReplicationVBMasterCheck+0x2b0 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/factory/xdcr_factory.go:933 # 0x9cb508 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).runP2PProtocol+0xe8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:376 # 0x9c9f8c github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).Start+0x76c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:274 # 0xa2f317 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineManager).StartPipeline+0x4d7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:464 # 0xa39524 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).update+0x424 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1899 # 0xa379c8 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run.func1+0x5c8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1610 # 0x48d321 sync.(*Once).doSlow+0xc1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:74 # 0xa373c6 sync.(*Once).Do+0x46 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:65 # 0xa37394 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run+0x14 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1588 11 @ 0x43f4b6 0x44fa7e 0xda0d65 0x471e01 # 0xda0d64 github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).run.func1+0x84 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1143 11 @ 0x43f4b6 0x450b6f 0x450b46 0x46dc46 0x48cf85 0xd9e31b 0xd9e304 0xda2771 0xda0ca5 0x471e01 # 0x46dc45 sync.runtime_SemacquireMutex+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:77 # 0x48cf84 sync.(*Mutex).lockSlow+0x164 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:171 # 0xd9e31a sync.(*Mutex).Lock+0x67a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:90 # 0xd9e303 github.com/couchbase/goxdcr/service_impl.(*BucketTopologyService).getHighSeqnosUpdater.func1+0x663 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:799 # 0xda2770 github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).updateOnce+0x570 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1362 # 0xda0ca4 github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).run.func2+0x1e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1176 11 @ 0x43f4b6 0x46deec 0x46decc 0x48b12c 0x96a170 0x97c3b3 0x8dc6bb 0x8dc47b 0x97bc8d 0x978b5c 0x978b42 0x978ed6 0x471e01 # 0x46decb sync.runtime_notifyListWait+0x12b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:527 # 0x48b12b sync.(*Cond).Wait+0x8b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/cond.go:70 # 0x96a16f github.com/couchbase/goxdcr/metadata_svc.(*BucketManifestGetter).GetManifest+0xef /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/bucketManifestGetter.go:87 # 0x97c3b2 github.com/couchbase/goxdcr/metadata_svc.(*CollectionsManifestAgent).refreshSourceCustom.func1+0x32 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/collections_manifest_service.go:953 # 0x8dc6ba github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithOriginalError+0xba /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2587 # 0x8dc47a github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutor+0x3a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2575 # 0x97bc8c github.com/couchbase/goxdcr/metadata_svc.(*CollectionsManifestAgent).refreshSourceCustom+0x12c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/collections_manifest_service.go:960 # 0x978b5b github.com/couchbase/goxdcr/metadata_svc.(*CollectionsManifestAgent).refreshSource+0x3b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/collections_manifest_service.go:937 # 0x978b41 github.com/couchbase/goxdcr/metadata_svc.(*CollectionsManifestAgent).refreshAndNotify+0x21 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/collections_manifest_service.go:464 # 0x978ed5 github.com/couchbase/goxdcr/metadata_svc.(*CollectionsManifestAgent).runPeriodicRefresh+0x95 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/collections_manifest_service.go:508 10 @ 0x43f4b6 0x44fa7e 0x7e727f 0x9e1ec8 0x471e01 # 0x7e727e github.com/couchbase/goxdcr/base.ExecWithTimeout+0x19e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/simple_utils.go:56 # 0x9e1ec7 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).initConnBg+0x67 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:538 9 @ 0x43f4b6 0x450b6f 0x450b46 0x46db87 0x4dcdd2 0x4e06f2 0x4e0703 0x4e9465 0x4e945b 0x5118e6 0x50ef6a 0x50f0dd 0x5121fc 0x512597 0xa237d6 0xa25d88 0x471e01 # 0x46db86 internal/poll.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:67 # 0x4dcdd1 internal/poll.(*fdMutex).rwlock+0xd1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:154 # 0x4e06f1 internal/poll.(*FD).writeLock+0x71 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:239 # 0x4e0702 internal/poll.(*FD).Write+0x82 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:370 # 0x4e9464 os.(*File).write+0x64 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file_posix.go:48 # 0x4e945a os.(*File).Write+0x5a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file.go:175 # 0x5118e5 github.com/couchbase/goxdcr/log.(*LogWriter).Write+0x25 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:94 # 0x50ef69 log.(*Logger).Output+0x349 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:194 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0x512596 github.com/couchbase/goxdcr/log.(*CommonLogger).Infof+0x36 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:193 # 0xa237d5 github.com/couchbase/goxdcr/pipeline_svc.(*TopologyChangeDetectorSvc).handleTargetTopologyChange+0x475 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/topology_change_detector.go:350 # 0xa25d87 github.com/couchbase/goxdcr/pipeline_svc.(*TopologyChangeDetectorSvc).monitorTarget.func1+0x327 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/topology_change_detector.go:610 9 @ 0x43f4b6 0x46deec 0x46decc 0x48b12c 0x96a216 0x97c3b3 0x8dc6bb 0x8dc47b 0x97bc8d 0x978b5c 0x978b42 0x978ed6 0x471e01 # 0x46decb sync.runtime_notifyListWait+0x12b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:527 # 0x48b12b sync.(*Cond).Wait+0x8b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/cond.go:70 # 0x96a215 github.com/couchbase/goxdcr/metadata_svc.(*BucketManifestGetter).GetManifest+0x195 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/bucketManifestGetter.go:93 # 0x97c3b2 github.com/couchbase/goxdcr/metadata_svc.(*CollectionsManifestAgent).refreshSourceCustom.func1+0x32 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/collections_manifest_service.go:953 # 0x8dc6ba github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithOriginalError+0xba /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2587 # 0x8dc47a github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutor+0x3a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2575 # 0x97bc8c github.com/couchbase/goxdcr/metadata_svc.(*CollectionsManifestAgent).refreshSourceCustom+0x12c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/collections_manifest_service.go:960 # 0x978b5b github.com/couchbase/goxdcr/metadata_svc.(*CollectionsManifestAgent).refreshSource+0x3b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/collections_manifest_service.go:937 # 0x978b41 github.com/couchbase/goxdcr/metadata_svc.(*CollectionsManifestAgent).refreshAndNotify+0x21 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/collections_manifest_service.go:464 # 0x978ed5 github.com/couchbase/goxdcr/metadata_svc.(*CollectionsManifestAgent).runPeriodicRefresh+0x95 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/collections_manifest_service.go:508 8 @ 0x43f4b6 0x44fa7e 0x7e727f 0x9c821d 0x9ca753 0xa2f318 0xa39525 0xa379c9 0x48d322 0xa373c7 0xa37395 0x471e01 # 0x7e727e github.com/couchbase/goxdcr/base.ExecWithTimeout+0x19e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/simple_utils.go:56 # 0x9c821c github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPartsWithTimeout+0x13c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:175 # 0x9ca752 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).Start+0xf32 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:326 # 0xa2f317 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineManager).StartPipeline+0x4d7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:464 # 0xa39524 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).update+0x424 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1899 # 0xa379c8 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run.func1+0x5c8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1610 # 0x48d321 sync.(*Once).doSlow+0xc1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:74 # 0xa373c6 sync.(*Once).Do+0x46 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:65 # 0xa37394 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run+0x14 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1588 8 @ 0x43f4b6 0x46deec 0x46decc 0x48b12c 0x9e0346 0x9fa830 0xa016d5 0x471e01 # 0x46decb sync.runtime_notifyListWait+0x12b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:527 # 0x48b12b sync.(*Cond).Wait+0x8b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/cond.go:70 # 0x9e0345 github.com/couchbase/goxdcr/pipeline_svc.(*checkpointSyncHelper).disableCkptAndWait+0x85 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:255 # 0x9fa82f github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).stopTheWorldAndMergeCkpts+0x16f /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:2746 # 0xa016d4 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).periodicMergerImpl+0x194 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:3542 7 @ 0x43f4b6 0x437d57 0x46c309 0x4dd352 0x4de739 0x4de727 0x5d7749 0x5e78a5 0x72640e 0x69315f 0x693d4f 0x6c33c5 0x6c2e28 0x6c30f6 0x71b29c 0x71b165 0x72ab02 0x6827e8 0x6823db 0x682018 0x95e2e5 0x471e01 # 0x46c308 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dd351 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4de738 internal/poll.(*pollDesc).waitRead+0x298 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4de726 internal/poll.(*FD).Read+0x286 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:167 # 0x5d7748 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e78a4 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x72640d net/http.(*persistConn).Read+0x4d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:1943 # 0x69315e bufio.(*Reader).fill+0xfe /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/bufio/bufio.go:106 # 0x693d4e bufio.(*Reader).ReadSlice+0x2e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/bufio/bufio.go:372 # 0x6c33c4 net/http/internal.readChunkLine+0x24 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/internal/chunked.go:129 # 0x6c2e27 net/http/internal.(*chunkedReader).beginChunk+0x27 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/internal/chunked.go:48 # 0x6c30f5 net/http/internal.(*chunkedReader).Read+0x155 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/internal/chunked.go:98 # 0x71b29b net/http.(*body).readLocked+0x3b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transfer.go:839 # 0x71b164 net/http.(*body).Read+0x124 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transfer.go:831 # 0x72ab01 net/http.(*bodyEOFSignal).Read+0x141 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:2792 # 0x6827e7 encoding/json.(*Decoder).refill+0x187 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/encoding/json/stream.go:165 # 0x6823da encoding/json.(*Decoder).readValue+0xba /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/encoding/json/stream.go:140 # 0x682017 encoding/json.(*Decoder).Decode+0x77 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/encoding/json/stream.go:63 # 0x95e2e4 github.com/couchbase/cbauth/metakv.doRunObserveChildren.func1+0xc4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:284 7 @ 0x43f4b6 0x44fa7e 0x727e05 0x471e01 # 0x727e04 net/http.(*persistConn).readLoop+0xd84 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:2227 7 @ 0x43f4b6 0x44fa7e 0x729d94 0x71dcba 0x705e59 0x6c6977 0x6c61fb 0x6c84fb 0x8d9413 0x8d940b 0x8d8693 0x8dfa27 0x976a7d 0x969f96 0x471e01 # 0x729d93 net/http.(*persistConn).roundTrip+0x993 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:2638 # 0x71dcb9 net/http.(*Transport).roundTrip+0x7f9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:603 # 0x705e58 net/http.(*Transport).RoundTrip+0x18 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/roundtrip.go:17 # 0x6c6976 net/http.send+0x5f6 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:252 # 0x6c61fa net/http.(*Client).send+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:176 # 0x6c84fa net/http.(*Client).do+0x8fa /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:716 # 0x8d9412 net/http.(*Client).Do+0x52 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:582 # 0x8d940a github.com/couchbase/goxdcr/utils.(*Utilities).doRestCall+0x4a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2234 # 0x8d8692 github.com/couchbase/goxdcr/utils.(*Utilities).QueryRestApiWithAuth+0x1b2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2096 # 0x8dfa26 github.com/couchbase/goxdcr/utils.(*Utilities).GetCollectionsManifest+0x226 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2926 # 0x976a7c github.com/couchbase/goxdcr/metadata_svc.(*CollectionsManifestService).CollectionManifestGetter+0x21c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/collections_manifest_service.go:180 # 0x969f95 github.com/couchbase/goxdcr/metadata_svc.(*BucketManifestGetter).runGetOp+0x55 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/bucketManifestGetter.go:60 7 @ 0x43f4b6 0x44fa7e 0x9213bf 0x471e01 # 0x9213be github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0x2de /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:995 7 @ 0x43f4b6 0x450b6f 0x450b46 0x46db27 0x48ea8b 0xdbfc5e 0xa1164a 0xa1163a 0xa1158f 0xa10933 0xa1010f 0xa0f225 0xa0e765 0x471e01 # 0x46db26 sync.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:62 # 0x48ea8a sync.(*WaitGroup).Wait+0x4a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/waitgroup.go:116 # 0xdbfc5d github.com/couchbase/goxdcr/service_impl.(*ThroughSeqnoTrackerSvc).GetThroughSeqnos+0x43d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/through_seqno_tracker_service.go:1538 # 0xa11649 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).GetThroughSeqnosFromTsService+0x89 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:2333 # 0xa11639 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).calculateChangesLeftAndDocsProcessedMainPipeline+0x79 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:664 # 0xa1158e github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).calculateChangesLeftAndDocProcessed+0x10e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:654 # 0xa10932 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).processCalculatedStats+0x72 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:548 # 0xa1010e github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).processRawStats+0x7ee /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:528 # 0xa0f224 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).updateStatsOnce+0x144 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:355 # 0xa0e764 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).updateStats+0x3e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:309 7 @ 0x43f4b6 0x450b6f 0x450b46 0x46db87 0x4dcdd2 0x4e06f2 0x4e0703 0x4e9465 0x4e945b 0x5118e6 0x50ef6a 0x50f0dd 0x5121fc 0xdbff25 0xdbfea5 0xdbfd18 0xa1164a 0xa1163a 0xa1158f 0xa10933 0xa1010f 0xa0f225 0xa0e765 0x471e01 # 0x46db86 internal/poll.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:67 # 0x4dcdd1 internal/poll.(*fdMutex).rwlock+0xd1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:154 # 0x4e06f1 internal/poll.(*FD).writeLock+0x71 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:239 # 0x4e0702 internal/poll.(*FD).Write+0x82 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:370 # 0x4e9464 os.(*File).write+0x64 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file_posix.go:48 # 0x4e945a os.(*File).Write+0x5a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file.go:175 # 0x5118e5 github.com/couchbase/goxdcr/log.(*LogWriter).Write+0x25 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:94 # 0x50ef69 log.(*Logger).Output+0x349 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:194 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0xdbff24 github.com/couchbase/goxdcr/log.(*CommonLogger).Warnf+0xc4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:189 # 0xdbfea4 github.com/couchbase/goxdcr/service_impl.(*ThroughSeqnoTrackerSvc).GetThroughSeqnos.func1+0x44 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/through_seqno_tracker_service.go:1509 # 0xdbfd17 github.com/couchbase/goxdcr/service_impl.(*ThroughSeqnoTrackerSvc).GetThroughSeqnos+0x4f7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/through_seqno_tracker_service.go:1546 # 0xa11649 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).GetThroughSeqnosFromTsService+0x89 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:2333 # 0xa11639 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).calculateChangesLeftAndDocsProcessedMainPipeline+0x79 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:664 # 0xa1158e github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).calculateChangesLeftAndDocProcessed+0x10e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:654 # 0xa10932 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).processCalculatedStats+0x72 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:548 # 0xa1010e github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).processRawStats+0x7ee /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:528 # 0xa0f224 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).updateStatsOnce+0x144 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:355 # 0xa0e764 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).updateStats+0x3e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:309 6 @ 0x43f4b6 0x437d57 0x46c309 0x4dd352 0x4de739 0x4de727 0x5d7749 0x5e78a5 0x4bb29a 0x68c179 0x68c143 0x745185 0x74514d 0x73d0f3 0x7436d3 0x8e0763 0xd9e5e3 0xda2771 0xda0ca5 0x471e01 # 0x46c308 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dd351 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4de738 internal/poll.(*pollDesc).waitRead+0x298 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4de726 internal/poll.(*FD).Read+0x286 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:167 # 0x5d7748 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e78a4 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x4bb299 io.ReadAtLeast+0x99 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:332 # 0x68c178 io.ReadFull+0xd8 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:351 # 0x68c142 github.com/couchbase/gomemcached.(*MCResponse).ReceiveWithBuf+0xa2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:169 # 0x745184 github.com/couchbase/gomemcached.(*MCResponse).Receive+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:156 # 0x74514c github.com/couchbase/gomemcached/client.getResponse+0x6c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/transport.go:35 # 0x73d0f2 github.com/couchbase/gomemcached/client.(*Client).Receive+0x52 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:333 # 0x7436d2 github.com/couchbase/gomemcached/client.(*Client).GetAllVbSeqnos+0xb2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:1559 # 0x8e0762 github.com/couchbase/goxdcr/utils.(*Utilities).GetHighSeqNos+0x2c2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:3031 # 0xd9e5e2 github.com/couchbase/goxdcr/service_impl.(*BucketTopologyService).getHighSeqnosUpdater.func1+0x942 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:814 # 0xda2770 github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).updateOnce+0x570 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1362 # 0xda0ca4 github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).run.func2+0x1e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1176 6 @ 0x43f4b6 0x450b6f 0x450b46 0x46db87 0x4dcdd2 0x4e06f2 0x4e0703 0x4e9465 0x4e945b 0x5118e6 0x50ef6a 0x50f0dd 0x5121fc 0x512597 0xa22ce4 0xa25266 0x471e01 # 0x46db86 internal/poll.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:67 # 0x4dcdd1 internal/poll.(*fdMutex).rwlock+0xd1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:154 # 0x4e06f1 internal/poll.(*FD).writeLock+0x71 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:239 # 0x4e0702 internal/poll.(*FD).Write+0x82 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:370 # 0x4e9464 os.(*File).write+0x64 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file_posix.go:48 # 0x4e945a os.(*File).Write+0x5a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file.go:175 # 0x5118e5 github.com/couchbase/goxdcr/log.(*LogWriter).Write+0x25 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:94 # 0x50ef69 log.(*Logger).Output+0x349 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:194 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0x512596 github.com/couchbase/goxdcr/log.(*CommonLogger).Infof+0x36 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:193 # 0xa22ce3 github.com/couchbase/goxdcr/pipeline_svc.(*TopologyChangeDetectorSvc).handleSourceTopologyChange+0x5a3 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/topology_change_detector.go:302 # 0xa25265 github.com/couchbase/goxdcr/pipeline_svc.(*TopologyChangeDetectorSvc).monitorSource.func1+0x3a5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/topology_change_detector.go:526 5 @ 0x43f4b6 0x450b6f 0x450b46 0x46db87 0x4dcdd2 0x4e06f2 0x4e0703 0x4e9465 0x4e945b 0x5118e6 0x50ef6a 0x50f0dd 0x5121fc 0x7e72ad 0x7e728a 0x9c821d 0x9ca753 0xa2f318 0xa39525 0xa379c9 0x48d322 0xa373c7 0xa37395 0x471e01 # 0x46db86 internal/poll.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:67 # 0x4dcdd1 internal/poll.(*fdMutex).rwlock+0xd1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:154 # 0x4e06f1 internal/poll.(*FD).writeLock+0x71 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:239 # 0x4e0702 internal/poll.(*FD).Write+0x82 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:370 # 0x4e9464 os.(*File).write+0x64 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file_posix.go:48 # 0x4e945a os.(*File).Write+0x5a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file.go:175 # 0x5118e5 github.com/couchbase/goxdcr/log.(*LogWriter).Write+0x25 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:94 # 0x50ef69 log.(*Logger).Output+0x349 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:194 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0x7e72ac github.com/couchbase/goxdcr/log.(*CommonLogger).Infof+0x1cc /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:193 # 0x7e7289 github.com/couchbase/goxdcr/base.ExecWithTimeout+0x1a9 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/simple_utils.go:60 # 0x9c821c github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPartsWithTimeout+0x13c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:175 # 0x9ca752 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).Start+0xf32 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:326 # 0xa2f317 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineManager).StartPipeline+0x4d7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:464 # 0xa39524 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).update+0x424 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1899 # 0xa379c8 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run.func1+0x5c8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1610 # 0x48d321 sync.(*Once).doSlow+0xc1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:74 # 0xa373c6 sync.(*Once).Do+0x46 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:65 # 0xa37394 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run+0x14 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1588 5 @ 0x43f4b6 0x450b6f 0x450b46 0x46dc46 0x48cf85 0x48e5d6 0x48e5b5 0x68e6ce 0x73f7cd 0x73efac 0x7e03f6 0x7e352d 0x8cc7f9 0x8ccd25 0x8dc6bb 0x8dc47b 0x8ccb08 0x8cd07f 0x9e3f5e 0x9e324f 0x48d322 0x9e2eb8 0x9e2e7e 0x9e2085 0x7e7482 0x471e01 # 0x46dc45 sync.runtime_SemacquireMutex+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:77 # 0x48cf84 sync.(*Mutex).lockSlow+0x164 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:171 # 0x48e5d5 sync.(*Mutex).Lock+0x35 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:90 # 0x48e5b4 sync.(*RWMutex).Lock+0x14 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/rwmutex.go:147 # 0x68e6cd github.com/couchbase/goutils/logging.Infof+0x6d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goutils/logging/logger.go:289 # 0x73f7cc github.com/couchbase/gomemcached/client.(*Client).AuthPlain+0x10c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:757 # 0x73efab github.com/couchbase/gomemcached/client.(*Client).Auth+0xab /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:688 # 0x7e03f5 github.com/couchbase/goxdcr/base.authClient+0x75 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:236 # 0x7e352c github.com/couchbase/goxdcr/base.NewConn+0x1ec /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:612 # 0x8cc7f8 github.com/couchbase/goxdcr/utils.(*Utilities).GetMemcachedRawConn+0x1f8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:524 # 0x8ccd24 github.com/couchbase/goxdcr/utils.(*Utilities).GetRemoteMemcachedConnectionWFeatures.func1+0x104 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:538 # 0x8dc6ba github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithOriginalError+0xba /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2587 # 0x8dc47a github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutor+0x3a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2575 # 0x8ccb07 github.com/couchbase/goxdcr/utils.(*Utilities).GetRemoteMemcachedConnectionWFeatures+0x1c7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:554 # 0x8cd07e github.com/couchbase/goxdcr/utils.(*Utilities).GetRemoteMemcachedConnection+0x9e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:568 # 0x9e3f5d github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).getNewMemcachedClient+0x49d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:710 # 0x9e324e github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).initConnections.func1+0x30e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:621 # 0x48d321 sync.(*Once).doSlow+0xc1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:74 # 0x9e2eb7 sync.(*Once).Do+0x57 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:65 # 0x9e2e7d github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).initConnections+0x1d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:609 # 0x9e2084 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).initConnBg.func1+0x64 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:536 # 0x7e7481 github.com/couchbase/goxdcr/base.ExecWithTimeout.func1+0x21 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/simple_utils.go:48 5 @ 0x43f4b6 0x450b6f 0x450b46 0x46dc46 0x48cf85 0x50ecbc 0x50ec9b 0x50f0dd 0x5121fc 0x8dffef 0x8dff6f 0x8cc879 0x8ccd25 0x8dc6bb 0x8dc47b 0x8ccb08 0x8cc3d0 0x91d65d 0x91e29a 0x91f78a 0x9c92e2 0x9c8d5f 0x471e01 # 0x46dc45 sync.runtime_SemacquireMutex+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:77 # 0x48cf84 sync.(*Mutex).lockSlow+0x164 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:171 # 0x50ecbb sync.(*Mutex).Lock+0x9b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:90 # 0x50ec9a log.(*Logger).Output+0x7a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:175 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0x8dffee github.com/couchbase/goxdcr/log.(*CommonLogger).Warnf+0xee /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:189 # 0x8dff6e github.com/couchbase/goxdcr/utils.(*Utilities).StartDiagStopwatch.func1+0x6e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2960 # 0x8cc878 github.com/couchbase/goxdcr/utils.(*Utilities).GetMemcachedRawConn+0x278 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:528 # 0x8ccd24 github.com/couchbase/goxdcr/utils.(*Utilities).GetRemoteMemcachedConnectionWFeatures.func1+0x104 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:538 # 0x8dc6ba github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithOriginalError+0xba /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2587 # 0x8dc47a github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutor+0x3a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2575 # 0x8ccb07 github.com/couchbase/goxdcr/utils.(*Utilities).GetRemoteMemcachedConnectionWFeatures+0x1c7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:554 # 0x8cc3cf github.com/couchbase/goxdcr/utils.(*Utilities).GetMemcachedConnectionWFeatures+0x40f /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:509 # 0x91d65c github.com/couchbase/goxdcr/parts.(*DcpNozzle).initializeMemcachedClient+0x1fc /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:502 # 0x91e299 github.com/couchbase/goxdcr/parts.(*DcpNozzle).initialize+0x179 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:635 # 0x91f789 github.com/couchbase/goxdcr/parts.(*DcpNozzle).Start+0x189 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:790 # 0x9c92e1 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart+0x321 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:233 # 0x9c8d5e github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPartsWithTimeout.func1.1+0xde /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:153 5 @ 0x43f4b6 0x450b6f 0x450b46 0x46dc46 0x48cf85 0x50ecbc 0x50ec9b 0x50f0dd 0x5121fc 0x9ea897 0x9ea70b 0x471e01 # 0x46dc45 sync.runtime_SemacquireMutex+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:77 # 0x48cf84 sync.(*Mutex).lockSlow+0x164 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:171 # 0x50ecbb sync.(*Mutex).Lock+0x9b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:90 # 0x50ec9a log.(*Logger).Output+0x7a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:175 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0x9ea896 github.com/couchbase/goxdcr/log.(*CommonLogger).Infof+0x216 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:193 # 0x9ea70a github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).startSeqnoGetter+0x8a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1316 5 @ 0x43f4b6 0x450b6f 0x450b46 0x46dc46 0x48cf85 0x50ecbc 0x50ec9b 0x50f0dd 0x5121fc 0xa37b6e 0xa37ab3 0x48d322 0xa373c7 0xa37395 0x471e01 # 0x46dc45 sync.runtime_SemacquireMutex+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:77 # 0x48cf84 sync.(*Mutex).lockSlow+0x164 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:171 # 0x50ecbb sync.(*Mutex).Lock+0x9b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:90 # 0x50ec9a log.(*Logger).Output+0x7a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:175 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0xa37b6d github.com/couchbase/goxdcr/log.(*CommonLogger).Infof+0x76d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:193 # 0xa37ab2 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run.func1+0x6b2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1613 # 0x48d321 sync.(*Once).doSlow+0xc1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:74 # 0xa373c6 sync.(*Once).Do+0x46 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:65 # 0xa37394 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run+0x14 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1588 5 @ 0x48e625 0x935912 0x9363e5 0x90f4c2 0x921da2 0x471e01 # 0x48e624 sync.(*RWMutex).Lock+0x84 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/rwmutex.go:141 # 0x935911 github.com/couchbase/goxdcr/parts.(*Router).ComposeMCRequest+0x691 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/router.go:1549 # 0x9363e4 github.com/couchbase/goxdcr/parts.(*Router).Route+0x3e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/router.go:1663 # 0x90f4c1 github.com/couchbase/goxdcr/connector.(*Router).Forward+0x141 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/connector/router.go:113 # 0x921da1 github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0xcc1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1102 5 @ 0x4c33a7 0x4c14a5 0x4de76e 0x4de756 0x4de4d6 0x5d7749 0x5e78a5 0x4bb29a 0x68c179 0x68c143 0x745185 0x74514d 0x73d0f3 0x9534ab 0x94c9c5 0x471e01 # 0x4c33a6 syscall.Syscall+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/syscall/syscall_linux.go:69 # 0x4c14a4 syscall.read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/syscall/zsyscall_linux_amd64.go:711 # 0x4de76d syscall.Read+0x2cd /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/syscall/syscall_unix.go:178 # 0x4de755 internal/poll.ignoringEINTRIO+0x2b5 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:794 # 0x4de4d5 internal/poll.(*FD).Read+0x35 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:163 # 0x5d7748 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e78a4 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x4bb299 io.ReadAtLeast+0x99 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:332 # 0x68c178 io.ReadFull+0xd8 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:351 # 0x68c142 github.com/couchbase/gomemcached.(*MCResponse).ReceiveWithBuf+0xa2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:169 # 0x745184 github.com/couchbase/gomemcached.(*MCResponse).Receive+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:156 # 0x74514c github.com/couchbase/gomemcached/client.getResponse+0x6c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/transport.go:35 # 0x73d0f2 github.com/couchbase/gomemcached/client.(*Client).Receive+0x52 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:333 # 0x9534aa github.com/couchbase/goxdcr/parts.(*XmemNozzle).readFromClient+0x18a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3498 # 0x94c9c4 github.com/couchbase/goxdcr/parts.(*XmemNozzle).receiveResponse+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2618 4 @ 0x43f4b6 0x437d57 0x46c309 0x4dd352 0x4de739 0x4de727 0x5d7749 0x5e78a5 0x4bb29a 0x68c179 0x68c143 0x745185 0x74514d 0x73cb89 0x73f9f2 0x7e0593 0x7e352d 0x8cc7f9 0x8ccd25 0x8dc6bb 0x8dc47b 0x8ccb08 0x8cd07f 0x9e3f5e 0x9e324f 0x48d322 0x9e2eb8 0x9e2e7e 0x9e2085 0x7e7482 0x471e01 # 0x46c308 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dd351 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4de738 internal/poll.(*pollDesc).waitRead+0x298 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4de726 internal/poll.(*FD).Read+0x286 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:167 # 0x5d7748 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e78a4 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x4bb299 io.ReadAtLeast+0x99 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:332 # 0x68c178 io.ReadFull+0xd8 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:351 # 0x68c142 github.com/couchbase/gomemcached.(*MCResponse).ReceiveWithBuf+0xa2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:169 # 0x745184 github.com/couchbase/gomemcached.(*MCResponse).Receive+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:156 # 0x74514c github.com/couchbase/gomemcached/client.getResponse+0x6c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/transport.go:35 # 0x73cb88 github.com/couchbase/gomemcached/client.(*Client).Send+0x68 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:280 # 0x73f9f1 github.com/couchbase/gomemcached/client.(*Client).SelectBucket+0x91 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:767 # 0x7e0592 github.com/couchbase/goxdcr/base.authClient+0x212 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:251 # 0x7e352c github.com/couchbase/goxdcr/base.NewConn+0x1ec /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:612 # 0x8cc7f8 github.com/couchbase/goxdcr/utils.(*Utilities).GetMemcachedRawConn+0x1f8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:524 # 0x8ccd24 github.com/couchbase/goxdcr/utils.(*Utilities).GetRemoteMemcachedConnectionWFeatures.func1+0x104 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:538 # 0x8dc6ba github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithOriginalError+0xba /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2587 # 0x8dc47a github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutor+0x3a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2575 # 0x8ccb07 github.com/couchbase/goxdcr/utils.(*Utilities).GetRemoteMemcachedConnectionWFeatures+0x1c7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:554 # 0x8cd07e github.com/couchbase/goxdcr/utils.(*Utilities).GetRemoteMemcachedConnection+0x9e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:568 # 0x9e3f5d github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).getNewMemcachedClient+0x49d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:710 # 0x9e324e github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).initConnections.func1+0x30e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:621 # 0x48d321 sync.(*Once).doSlow+0xc1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:74 # 0x9e2eb7 sync.(*Once).Do+0x57 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:65 # 0x9e2e7d github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).initConnections+0x1d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:609 # 0x9e2084 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).initConnBg.func1+0x64 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:536 # 0x7e7481 github.com/couchbase/goxdcr/base.ExecWithTimeout.func1+0x21 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/simple_utils.go:48 4 @ 0x43f4b6 0x437d57 0x46c309 0x4dd352 0x4de739 0x4de727 0x5d7749 0x5e78a5 0x4bb29a 0x68c179 0x68c143 0x745185 0x74514d 0x73d0f3 0x747fdd 0x748c9d 0x7483af 0x91db8b 0x91e6c7 0x91f78a 0x9c92e2 0x9c8d5f 0x471e01 # 0x46c308 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dd351 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4de738 internal/poll.(*pollDesc).waitRead+0x298 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4de726 internal/poll.(*FD).Read+0x286 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:167 # 0x5d7748 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e78a4 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x4bb299 io.ReadAtLeast+0x99 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:332 # 0x68c178 io.ReadFull+0xd8 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:351 # 0x68c142 github.com/couchbase/gomemcached.(*MCResponse).ReceiveWithBuf+0xa2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:169 # 0x745184 github.com/couchbase/gomemcached.(*MCResponse).Receive+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:156 # 0x74514c github.com/couchbase/gomemcached/client.getResponse+0x6c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/transport.go:35 # 0x73d0f2 github.com/couchbase/gomemcached/client.(*Client).Receive+0x52 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:333 # 0x747fdc github.com/couchbase/gomemcached/client.sendMcRequestSync+0x3c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/upr_feed.go:450 # 0x748c9c github.com/couchbase/gomemcached/client.(*UprFeed).uprOpen+0x5dc /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/upr_feed.go:567 # 0x7483ae github.com/couchbase/gomemcached/client.(*UprFeed).UprOpenWithFeatures+0x8e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/upr_feed.go:481 # 0x91db8a github.com/couchbase/goxdcr/parts.(*DcpNozzle).initializeUprFeed+0x2ca /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:575 # 0x91e6c6 github.com/couchbase/goxdcr/parts.(*DcpNozzle).initialize+0x5a6 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:739 # 0x91f789 github.com/couchbase/goxdcr/parts.(*DcpNozzle).Start+0x189 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:790 # 0x9c92e1 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart+0x321 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:233 # 0x9c8d5e github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPartsWithTimeout.func1.1+0xde /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:153 4 @ 0x43f4b6 0x44fa7e 0x721d46 0x71dc5e 0x705e59 0x6c6977 0x6c61fb 0x6c84fb 0x8d9413 0x8d940b 0x8d8693 0x8dfa27 0x976a7d 0x969f96 0x471e01 # 0x721d45 net/http.(*Transport).getConn+0x5c5 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:1382 # 0x71dc5d net/http.(*Transport).roundTrip+0x79d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:590 # 0x705e58 net/http.(*Transport).RoundTrip+0x18 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/roundtrip.go:17 # 0x6c6976 net/http.send+0x5f6 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:252 # 0x6c61fa net/http.(*Client).send+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:176 # 0x6c84fa net/http.(*Client).do+0x8fa /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:716 # 0x8d9412 net/http.(*Client).Do+0x52 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:582 # 0x8d940a github.com/couchbase/goxdcr/utils.(*Utilities).doRestCall+0x4a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2234 # 0x8d8692 github.com/couchbase/goxdcr/utils.(*Utilities).QueryRestApiWithAuth+0x1b2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2096 # 0x8dfa26 github.com/couchbase/goxdcr/utils.(*Utilities).GetCollectionsManifest+0x226 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2926 # 0x976a7c github.com/couchbase/goxdcr/metadata_svc.(*CollectionsManifestService).CollectionManifestGetter+0x21c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/collections_manifest_service.go:180 # 0x969f95 github.com/couchbase/goxdcr/metadata_svc.(*BucketManifestGetter).runGetOp+0x55 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/bucketManifestGetter.go:60 4 @ 0x43f4b6 0x44fa7e 0x95dea9 0x95e3f8 0x95e3ba 0xde4388 0x471e01 # 0x95dea8 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x428 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:301 # 0x95e3f7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:259 # 0x95e3b9 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x19 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:389 # 0xde4387 github.com/couchbase/goxdcr/replication_manager.(*MetakvChangeListener).observeChildren+0x87 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/replication_manager/metakv_change_listener.go:171 4 @ 0x43f4b6 0x450b6f 0x450b46 0x46dc46 0x48cf85 0x48e5d6 0x48e5b5 0x68e6ce 0x73f7cd 0x73efac 0x7e03f6 0x7e352d 0x7e0095 0x7e001b 0x955822 0x8dc98a 0x955345 0x94afce 0x94bf6c 0x93cb4b 0x9c92e2 0x9c973a 0x471e01 # 0x46dc45 sync.runtime_SemacquireMutex+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:77 # 0x48cf84 sync.(*Mutex).lockSlow+0x164 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:171 # 0x48e5d5 sync.(*Mutex).Lock+0x35 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:90 # 0x48e5b4 sync.(*RWMutex).Lock+0x14 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/rwmutex.go:147 # 0x68e6cd github.com/couchbase/goutils/logging.Infof+0x6d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goutils/logging/logger.go:289 # 0x73f7cc github.com/couchbase/gomemcached/client.(*Client).AuthPlain+0x10c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:757 # 0x73efab github.com/couchbase/gomemcached/client.(*Client).Auth+0xab /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:688 # 0x7e03f5 github.com/couchbase/goxdcr/base.authClient+0x75 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:236 # 0x7e352c github.com/couchbase/goxdcr/base.NewConn+0x1ec /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:612 # 0x7e0094 github.com/couchbase/goxdcr/base.(*connPool).newConn+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:208 # 0x7e001a github.com/couchbase/goxdcr/base.(*connPool).GetNew+0x1a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:204 # 0x955821 github.com/couchbase/goxdcr/parts.(*XmemNozzle).getClientWithRetry.func1+0x2a1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3774 # 0x8dc989 github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithFinishSignal+0x109 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2615 # 0x955344 github.com/couchbase/goxdcr/parts.(*XmemNozzle).getClientWithRetry+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3777 # 0x94afcd github.com/couchbase/goxdcr/parts.(*XmemNozzle).initializeConnection+0x38d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2406 # 0x94bf6b github.com/couchbase/goxdcr/parts.(*XmemNozzle).initialize+0x72b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2550 # 0x93cb4a github.com/couchbase/goxdcr/parts.(*XmemNozzle).Start+0x1ca /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:807 # 0x9c92e1 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart+0x321 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:233 # 0x9c9739 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart.func1+0xb9 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:209 4 @ 0x43f4b6 0x450b6f 0x450b46 0x46dc46 0x48cf85 0x50ecbc 0x50ec9b 0x50f0dd 0x5121fc 0x7e7830 0x7e77cd 0x9b90e5 0x9b9c9d 0x710689 0x711e36 0x70d812 0x471e01 # 0x46dc45 sync.runtime_SemacquireMutex+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:77 # 0x48cf84 sync.(*Mutex).lockSlow+0x164 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:171 # 0x50ecbb sync.(*Mutex).Lock+0x9b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:90 # 0x50ec9a log.(*Logger).Output+0x7a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:175 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0x7e782f github.com/couchbase/goxdcr/log.(*CommonLogger).Infof+0x36f /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:193 # 0x7e77cc github.com/couchbase/goxdcr/base.ExecWithTimeout3+0x30c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/simple_utils.go:126 # 0x9b90e4 github.com/couchbase/goxdcr/adminport.(*httpServer).systemHandler+0x1c4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/adminport/admin_httpd.go:131 # 0x9b9c9c github.com/couchbase/goxdcr/adminport.(*Handler).ServeHTTP+0x13c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/adminport/admin_httpd.go:204 # 0x710688 net/http.(*ServeMux).ServeHTTP+0x148 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/server.go:2500 # 0x711e35 net/http.serverHandler.ServeHTTP+0x315 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/server.go:2936 # 0x70d811 net/http.(*conn).serve+0x611 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/server.go:1995 4 @ 0x43f4b6 0x450b6f 0x450b46 0x46dc46 0x48cf85 0x50ecbc 0x50ec9b 0x50f23a 0x512329 0xa0f3ee 0xa0f3d0 0xa0ee3b 0x471e01 # 0x46dc45 sync.runtime_SemacquireMutex+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:77 # 0x48cf84 sync.(*Mutex).lockSlow+0x164 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:171 # 0x50ecbb sync.(*Mutex).Lock+0x9b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:90 # 0x50ec9a log.(*Logger).Output+0x7a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:175 # 0x50f239 log.(*Logger).Println+0x59 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:222 # 0x512328 github.com/couchbase/goxdcr/log.(*CommonLogger).logMsg+0xc8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:176 # 0xa0f3ed github.com/couchbase/goxdcr/log.(*CommonLogger).Info+0x6d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:217 # 0xa0f3cf github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).logStatsOnce+0x4f /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:369 # 0xa0ee3a github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).logStats+0x23a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:336 4 @ 0x745b85 0x7454d8 0x74b2e5 0x471e01 # 0x745b84 github.com/couchbase/gomemcached/client.(*UprEvent).PopulateFieldsBasedOnStreamType+0x2c4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/upr_event.go:168 # 0x7454d7 github.com/couchbase/gomemcached/client.makeUprEvent+0x177 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/upr_event.go:111 # 0x74b2e4 github.com/couchbase/gomemcached/client.(*UprFeed).runFeed+0x8a4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/upr_feed.go:945 3 @ 0x41f13b 0x413308 0x419068 0x419053 0x93667a 0x90f4c2 0x921da2 0x471e01 # 0x936679 github.com/couchbase/goxdcr/parts.(*Router).Route+0x679 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/router.go:1685 # 0x90f4c1 github.com/couchbase/goxdcr/connector.(*Router).Forward+0x141 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/connector/router.go:113 # 0x921da1 github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0xcc1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1102 3 @ 0x43f4b6 0x437d57 0x46c309 0x4dd352 0x4de739 0x4de727 0x5d7749 0x5e78a5 0x4bb29a 0x68c179 0x68c143 0x745185 0x74514d 0x73cb89 0x73d5e5 0x8cd5f8 0x8cce9c 0x8dc6bb 0x8dc47b 0x8ccb08 0x8cc3d0 0x91d65d 0x91e29a 0x91f78a 0x9c92e2 0x9c8d5f 0x471e01 # 0x46c308 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dd351 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4de738 internal/poll.(*pollDesc).waitRead+0x298 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4de726 internal/poll.(*FD).Read+0x286 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:167 # 0x5d7748 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e78a4 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x4bb299 io.ReadAtLeast+0x99 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:332 # 0x68c178 io.ReadFull+0xd8 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:351 # 0x68c142 github.com/couchbase/gomemcached.(*MCResponse).ReceiveWithBuf+0xa2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:169 # 0x745184 github.com/couchbase/gomemcached.(*MCResponse).Receive+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:156 # 0x74514c github.com/couchbase/gomemcached/client.getResponse+0x6c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/transport.go:35 # 0x73cb88 github.com/couchbase/gomemcached/client.(*Client).Send+0x68 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:280 # 0x73d5e4 github.com/couchbase/gomemcached/client.(*Client).EnableFeatures+0x1c4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:393 # 0x8cd5f7 github.com/couchbase/goxdcr/utils.(*Utilities).SendHELOWithFeatures+0x217 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:624 # 0x8cce9b github.com/couchbase/goxdcr/utils.(*Utilities).GetRemoteMemcachedConnectionWFeatures.func1+0x27b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:544 # 0x8dc6ba github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithOriginalError+0xba /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2587 # 0x8dc47a github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutor+0x3a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2575 # 0x8ccb07 github.com/couchbase/goxdcr/utils.(*Utilities).GetRemoteMemcachedConnectionWFeatures+0x1c7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:554 # 0x8cc3cf github.com/couchbase/goxdcr/utils.(*Utilities).GetMemcachedConnectionWFeatures+0x40f /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:509 # 0x91d65c github.com/couchbase/goxdcr/parts.(*DcpNozzle).initializeMemcachedClient+0x1fc /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:502 # 0x91e299 github.com/couchbase/goxdcr/parts.(*DcpNozzle).initialize+0x179 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:635 # 0x91f789 github.com/couchbase/goxdcr/parts.(*DcpNozzle).Start+0x189 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:790 # 0x9c92e1 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart+0x321 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:233 # 0x9c8d5e github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPartsWithTimeout.func1.1+0xde /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:153 3 @ 0x43f4b6 0x437d57 0x46c309 0x4dd352 0x4de739 0x4de727 0x5d7749 0x5e78a5 0x4bb29a 0x68c179 0x68c143 0x745185 0x74514d 0x73cb89 0x73f8fd 0x73efac 0x7e03f6 0x7e352d 0x7e0095 0x7e001b 0x955822 0x8dc98a 0x955345 0x94b06f 0x94bf6c 0x93cb4b 0x9c92e2 0x9c973a 0x471e01 # 0x46c308 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dd351 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4de738 internal/poll.(*pollDesc).waitRead+0x298 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4de726 internal/poll.(*FD).Read+0x286 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:167 # 0x5d7748 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e78a4 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x4bb299 io.ReadAtLeast+0x99 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:332 # 0x68c178 io.ReadFull+0xd8 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:351 # 0x68c142 github.com/couchbase/gomemcached.(*MCResponse).ReceiveWithBuf+0xa2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:169 # 0x745184 github.com/couchbase/gomemcached.(*MCResponse).Receive+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:156 # 0x74514c github.com/couchbase/gomemcached/client.getResponse+0x6c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/transport.go:35 # 0x73cb88 github.com/couchbase/gomemcached/client.(*Client).Send+0x68 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:280 # 0x73f8fc github.com/couchbase/gomemcached/client.(*Client).AuthPlain+0x23c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:759 # 0x73efab github.com/couchbase/gomemcached/client.(*Client).Auth+0xab /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:688 # 0x7e03f5 github.com/couchbase/goxdcr/base.authClient+0x75 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:236 # 0x7e352c github.com/couchbase/goxdcr/base.NewConn+0x1ec /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:612 # 0x7e0094 github.com/couchbase/goxdcr/base.(*connPool).newConn+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:208 # 0x7e001a github.com/couchbase/goxdcr/base.(*connPool).GetNew+0x1a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:204 # 0x955821 github.com/couchbase/goxdcr/parts.(*XmemNozzle).getClientWithRetry.func1+0x2a1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3774 # 0x8dc989 github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithFinishSignal+0x109 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2615 # 0x955344 github.com/couchbase/goxdcr/parts.(*XmemNozzle).getClientWithRetry+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3777 # 0x94b06e github.com/couchbase/goxdcr/parts.(*XmemNozzle).initializeConnection+0x42e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2415 # 0x94bf6b github.com/couchbase/goxdcr/parts.(*XmemNozzle).initialize+0x72b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2550 # 0x93cb4a github.com/couchbase/goxdcr/parts.(*XmemNozzle).Start+0x1ca /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:807 # 0x9c92e1 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart+0x321 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:233 # 0x9c9739 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart.func1+0xb9 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:209 3 @ 0x43f4b6 0x437d57 0x46c309 0x4dd352 0x5d9179 0x5d915d 0x5d9181 0x5ecb05 0x5ec632 0x5e3b98 0x5f02cf 0x5f0169 0x5ce992 0x5ce2e5 0x5cd953 0x5cd36c 0x5ccc07 0x5ccc08 0x73c52a 0x73c5ae 0x7e33e8 0x7e0095 0x7e001b 0x955822 0x8dc98a 0x955345 0x94afce 0x94bf6c 0x93cb4b 0x9c92e2 0x9c973a 0x471e01 # 0x46c308 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dd351 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x5d9178 internal/poll.(*pollDesc).waitWrite+0x798 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:93 # 0x5d915c internal/poll.(*FD).WaitWrite+0x77c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:741 # 0x5d9180 net.(*netFD).connect+0x7a0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_unix.go:141 # 0x5ecb04 net.(*netFD).dial+0x3c4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/sock_posix.go:151 # 0x5ec631 net.socket+0x2d1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/sock_posix.go:70 # 0x5e3b97 net.internetSocket+0xf7 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/ipsock_posix.go:142 # 0x5f02ce net.(*sysDialer).doDialTCP+0xee /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/tcpsock_posix.go:74 # 0x5f0168 net.(*sysDialer).dialTCP+0x68 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/tcpsock_posix.go:64 # 0x5ce991 net.(*sysDialer).dialSingle+0x1f1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:580 # 0x5ce2e4 net.(*sysDialer).dialSerial+0x244 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:548 # 0x5cd952 net.(*sysDialer).dialParallel+0x412 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:449 # 0x5cd36b net.(*Dialer).DialContext+0x70b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:440 # 0x5ccc06 net.(*Dialer).Dial+0x86 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:367 # 0x5ccc07 net.DialTimeout+0x87 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:349 # 0x73c529 github.com/couchbase/gomemcached/client.glob..func1+0x29 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:189 # 0x73c5ad github.com/couchbase/gomemcached/client.Connect+0x2d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:195 # 0x7e33e7 github.com/couchbase/goxdcr/base.NewConn+0xa7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:605 # 0x7e0094 github.com/couchbase/goxdcr/base.(*connPool).newConn+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:208 # 0x7e001a github.com/couchbase/goxdcr/base.(*connPool).GetNew+0x1a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:204 # 0x955821 github.com/couchbase/goxdcr/parts.(*XmemNozzle).getClientWithRetry.func1+0x2a1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3774 # 0x8dc989 github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithFinishSignal+0x109 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2615 # 0x955344 github.com/couchbase/goxdcr/parts.(*XmemNozzle).getClientWithRetry+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3777 # 0x94afcd github.com/couchbase/goxdcr/parts.(*XmemNozzle).initializeConnection+0x38d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2406 # 0x94bf6b github.com/couchbase/goxdcr/parts.(*XmemNozzle).initialize+0x72b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2550 # 0x93cb4a github.com/couchbase/goxdcr/parts.(*XmemNozzle).Start+0x1ca /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:807 # 0x9c92e1 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart+0x321 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:233 # 0x9c9739 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart.func1+0xb9 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:209 3 @ 0x43f4b6 0x44fa7e 0x729d94 0x71dcba 0x705e59 0x6c6977 0x6c61fb 0x6c84fb 0x8d9413 0x8d940b 0x8d8693 0x8d2965 0x9957ac 0xd9ac76 0xda2771 0xda0ca5 0x471e01 # 0x729d93 net/http.(*persistConn).roundTrip+0x993 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:2638 # 0x71dcb9 net/http.(*Transport).roundTrip+0x7f9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:603 # 0x705e58 net/http.(*Transport).RoundTrip+0x18 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/roundtrip.go:17 # 0x6c6976 net/http.send+0x5f6 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:252 # 0x6c61fa net/http.(*Client).send+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:176 # 0x6c84fa net/http.(*Client).do+0x8fa /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:716 # 0x8d9412 net/http.(*Client).Do+0x52 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:582 # 0x8d940a github.com/couchbase/goxdcr/utils.(*Utilities).doRestCall+0x4a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2234 # 0x8d8692 github.com/couchbase/goxdcr/utils.(*Utilities).QueryRestApiWithAuth+0x1b2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2096 # 0x8d2964 github.com/couchbase/goxdcr/utils.(*Utilities).GetBucketInfo+0x324 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:1232 # 0x9957ab github.com/couchbase/goxdcr/metadata_svc.(*RemoteClusterAgent).RegisterBucketRequest.func1+0x34b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/remote_cluster_service.go:2124 # 0xd9ac75 github.com/couchbase/goxdcr/service_impl.(*BucketTopologyService).getOrCreateRemoteWatcher.func1+0x55 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:472 # 0xda2770 github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).updateOnce+0x570 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1362 # 0xda0ca4 github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).run.func2+0x1e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1176 3 @ 0x43f4b6 0x44fa7e 0x8e84df 0x471e01 # 0x8e84de github.com/couchbase/goxdcr/peerToPeer.(*HandlerCommon).ClearOpaqueBg+0x7e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/handlerCommon.go:148 3 @ 0x43f4b6 0x44fa7e 0x95dea9 0x95e3f8 0x95e3ba 0xd851fc 0xd7e224 0xd85147 0x471e01 # 0x95dea8 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x428 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:301 # 0x95e3f7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:259 # 0x95e3b9 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x19 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:389 # 0xd851fb github.com/couchbase/eventing-ee/evaluator/store.ObserveChanges.func1+0x7b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing-ee/evaluator/store/store.go:389 # 0xd7e223 github.com/couchbase/eventing-ee/evaluator/utils.ExponentialBackoffRetry+0x43 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing-ee/evaluator/utils/retry.go:57 # 0xd85146 github.com/couchbase/eventing-ee/evaluator/store.ObserveChanges+0x86 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing-ee/evaluator/store/store.go:387 3 @ 0x43f4b6 0x450b6f 0x450b46 0x46db27 0x48ea8b 0x8fc20c 0x471e01 # 0x46db26 sync.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:62 # 0x48ea8a sync.(*WaitGroup).Wait+0x4a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/waitgroup.go:116 # 0x8fc20b github.com/couchbase/goxdcr/peerToPeer.(*PeriodicPushHandler).handleRequest+0x66b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/periodicPushHandler.go:136 3 @ 0x43f4b6 0x450b6f 0x450b46 0x46db87 0x4dcdd2 0x4e06f2 0x4e0703 0x4e9465 0x4e945b 0x5118e6 0x50ef6a 0x50f0dd 0x5121fc 0x8cdb0b 0x8cda74 0x951e10 0x94b0e9 0x94bf6c 0x93cb4b 0x9c92e2 0x9c973a 0x471e01 # 0x46db86 internal/poll.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:67 # 0x4dcdd1 internal/poll.(*fdMutex).rwlock+0xd1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:154 # 0x4e06f1 internal/poll.(*FD).writeLock+0x71 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:239 # 0x4e0702 internal/poll.(*FD).Write+0x82 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:370 # 0x4e9464 os.(*File).write+0x64 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file_posix.go:48 # 0x4e945a os.(*File).Write+0x5a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file.go:175 # 0x5118e5 github.com/couchbase/goxdcr/log.(*LogWriter).Write+0x25 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:94 # 0x50ef69 log.(*Logger).Output+0x349 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:194 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0x8cdb0a github.com/couchbase/goxdcr/log.(*CommonLogger).Infof+0x72a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:193 # 0x8cda73 github.com/couchbase/goxdcr/utils.(*Utilities).SendHELOWithFeatures+0x693 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:661 # 0x951e0f github.com/couchbase/goxdcr/parts.(*XmemNozzle).sendHELO+0xcf /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3258 # 0x94b0e8 github.com/couchbase/goxdcr/parts.(*XmemNozzle).initializeConnection+0x4a8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2424 # 0x94bf6b github.com/couchbase/goxdcr/parts.(*XmemNozzle).initialize+0x72b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2550 # 0x93cb4a github.com/couchbase/goxdcr/parts.(*XmemNozzle).Start+0x1ca /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:807 # 0x9c92e1 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart+0x321 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:233 # 0x9c9739 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart.func1+0xb9 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:209 3 @ 0x43f4b6 0x450b6f 0x450b46 0x46dd06 0x48e611 0x7ffaa5 0x95329a 0x952689 0x94a32c 0x950c4b 0x939f54 0x9506c9 0x471e01 # 0x46dd05 sync.runtime_SemacquireRWMutex+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:87 # 0x48e610 sync.(*RWMutex).Lock+0x70 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/rwmutex.go:152 # 0x7ffaa4 github.com/couchbase/goxdcr/base.(*XmemClient).ReportOpSuccess+0x44 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/xmem_client.go:101 # 0x953299 github.com/couchbase/goxdcr/parts.(*XmemNozzle).writeToClientWithoutThrottling+0x299 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3466 # 0x952688 github.com/couchbase/goxdcr/parts.(*XmemNozzle).writeToClient+0x388 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3340 # 0x94a32b github.com/couchbase/goxdcr/parts.(*XmemNozzle).sendSingleSetMeta+0xab /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2334 # 0x950c4a github.com/couchbase/goxdcr/parts.(*XmemNozzle).resendIfTimeout+0x46a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3051 # 0x939f53 github.com/couchbase/goxdcr/parts.(*requestBuffer).modSlot+0x53 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:296 # 0x9506c8 github.com/couchbase/goxdcr/parts.(*XmemNozzle).checkAndRepairBufferMonitor+0x348 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2995 3 @ 0x450e3c 0x450e30 0x46dbd9 0x48d185 0x48d0e9 0x50ef8c 0x50f0dd 0x5121fc 0x9ea2e5 0x9ea193 0x9ea9d4 0x471e01 # 0x46dbd8 sync.runtime_Semrelease+0x18 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:72 # 0x48d184 sync.(*Mutex).unlockSlow+0x64 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:257 # 0x48d0e8 sync.(*Mutex).Unlock+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:223 # 0x50ef8b log.(*Logger).Output+0x36b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:195 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0x9ea2e4 github.com/couchbase/goxdcr/log.(*CommonLogger).Infof+0x204 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:193 # 0x9ea192 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).setTimestampForVB+0xb2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1285 # 0x9ea9d3 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).startSeqnoGetter+0x353 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1335 3 @ 0x48e7cd 0x93741d 0x93723a 0x936649 0x90f4c2 0x921da2 0x471e01 # 0x48e7cc sync.(*RWMutex).Unlock+0xac /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/rwmutex.go:198 # 0x93741c github.com/couchbase/goxdcr/parts.(*Router).prepareMcRequest+0xbc /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/router.go:1812 # 0x937239 github.com/couchbase/goxdcr/parts.(*Router).RouteCollection+0x4d9 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/router.go:1748 # 0x936648 github.com/couchbase/goxdcr/parts.(*Router).Route+0x648 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/router.go:1679 # 0x90f4c1 github.com/couchbase/goxdcr/connector.(*Router).Forward+0x141 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/connector/router.go:113 # 0x921da1 github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0xcc1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1102 3 @ 0x4c33a7 0x4c14a5 0x4de76e 0x4de756 0x4de4d6 0x5d7749 0x5e78a5 0x4bb29a 0x68b4dc 0x68b4ae 0x74ab8d 0x471e01 # 0x4c33a6 syscall.Syscall+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/syscall/syscall_linux.go:69 # 0x4c14a4 syscall.read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/syscall/zsyscall_linux_amd64.go:711 # 0x4de76d syscall.Read+0x2cd /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/syscall/syscall_unix.go:178 # 0x4de755 internal/poll.ignoringEINTRIO+0x2b5 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:794 # 0x4de4d5 internal/poll.(*FD).Read+0x35 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:163 # 0x5d7748 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e78a4 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x4bb299 io.ReadAtLeast+0x99 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:332 # 0x68b4db io.ReadFull+0x9b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:351 # 0x68b4ad github.com/couchbase/gomemcached.(*MCRequest).Receive+0x6d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_req.go:643 # 0x74ab8c github.com/couchbase/gomemcached/client.(*UprFeed).runFeed+0x14c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/upr_feed.go:904 3 @ 0x4c33a7 0x4c1665 0x4e0a05 0x4e09f7 0x4e0840 0x5d7fc9 0x5e7a85 0x9530c2 0x952689 0x94a32c 0x950c4b 0x939f54 0x9506c9 0x471e01 # 0x4c33a6 syscall.Syscall+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/syscall/syscall_linux.go:69 # 0x4c1664 syscall.write+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/syscall/zsyscall_linux_amd64.go:939 # 0x4e0a04 syscall.Write+0x384 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/syscall/syscall_unix.go:206 # 0x4e09f6 internal/poll.ignoringEINTRIO+0x376 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:794 # 0x4e083f internal/poll.(*FD).Write+0x1bf /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:383 # 0x5d7fc8 net.(*netFD).Write+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:96 # 0x5e7a84 net.(*conn).Write+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:195 # 0x9530c1 github.com/couchbase/goxdcr/parts.(*XmemNozzle).writeToClientWithoutThrottling+0xc1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3463 # 0x952688 github.com/couchbase/goxdcr/parts.(*XmemNozzle).writeToClient+0x388 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3340 # 0x94a32b github.com/couchbase/goxdcr/parts.(*XmemNozzle).sendSingleSetMeta+0xab /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2334 # 0x950c4a github.com/couchbase/goxdcr/parts.(*XmemNozzle).resendIfTimeout+0x46a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3051 # 0x939f53 github.com/couchbase/goxdcr/parts.(*requestBuffer).modSlot+0x53 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:296 # 0x9506c8 github.com/couchbase/goxdcr/parts.(*XmemNozzle).checkAndRepairBufferMonitor+0x348 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2995 3 @ 0x509a7f 0x7f3425 0x93665f 0x90f4c2 0x921da2 0x471e01 # 0x509a7e bytes.(*Buffer).Write+0xfe /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/bytes/buffer.go:166 # 0x7f3424 github.com/couchbase/goxdcr/base.(*WrappedMCRequest).ConstructUniqueKey+0x84 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/types.go:745 # 0x93665e github.com/couchbase/goxdcr/parts.(*Router).Route+0x65e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/router.go:1684 # 0x90f4c1 github.com/couchbase/goxdcr/connector.(*Router).Forward+0x141 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/connector/router.go:113 # 0x921da1 github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0xcc1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1102 3 @ 0x90e05d 0x921d09 0x471e01 # 0x90e05c github.com/couchbase/goxdcr/component.(*AbstractComponent).RaiseEvent+0xfc /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/component/abstract_component.go:95 # 0x921d08 github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0xc28 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1093 2 @ 0x41359b 0x921c79 0x921ce7 0x471e01 # 0x921c78 github.com/couchbase/goxdcr/common.NewEvent+0xb98 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/common/component.go:118 # 0x921ce6 github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0xc06 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1093 2 @ 0x41577f 0x90e887 0x471e01 # 0x90e886 github.com/couchbase/goxdcr/component.(*AsyncComponentEventListenerImpl).run+0xe6 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/component/async_listener.go:73 2 @ 0x41f13b 0x413308 0x413587 0x7453b5 0x74b2e5 0x471e01 # 0x7453b4 github.com/couchbase/gomemcached/client.makeUprEvent+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/upr_event.go:97 # 0x74b2e4 github.com/couchbase/gomemcached/client.(*UprFeed).runFeed+0x8a4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/upr_feed.go:945 2 @ 0x43f4b6 0x40c69d 0x40c198 0x728111 0x72abca 0x72abaf 0x6c9bd3 0x4bb985 0x4bc063 0x8d9645 0x8d95b2 0x8d9450 0x8da07e 0xda8dc9 0xda75e5 0x9eb9bd 0x9ea94c 0x471e01 # 0x728110 net/http.(*persistConn).readLoop.func4+0xb0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:2199 # 0x72abc9 net/http.(*bodyEOFSignal).condfn+0x209 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:2823 # 0x72abae net/http.(*bodyEOFSignal).Read+0x1ee /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:2799 # 0x6c9bd2 net/http.(*cancelTimerBody).Read+0x32 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:955 # 0x4bb984 io.(*LimitedReader).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:477 # 0x4bc062 io.ReadAll+0x102 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:701 # 0x8d9644 io/ioutil.ReadAll+0x184 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/ioutil/ioutil.go:27 # 0x8d95b1 github.com/couchbase/goxdcr/utils.(*Utilities).parseResponseBody+0xf1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2255 # 0x8d944f github.com/couchbase/goxdcr/utils.(*Utilities).doRestCall+0x8f /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2236 # 0x8da07d github.com/couchbase/goxdcr/utils.(*Utilities).InvokeRestWithRetryWithAuth+0x37d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2321 # 0xda8dc8 github.com/couchbase/goxdcr/service_impl.(*CAPIService).send_post+0x308 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/capi_service.go:291 # 0xda75e4 github.com/couchbase/goxdcr/service_impl.(*CAPIService).PreReplicate+0x144 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/capi_service.go:122 # 0x9eb9bc github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).getDataFromCkpts+0x4bc /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1444 # 0x9ea94b github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).startSeqnoGetter+0x2cb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1321 2 @ 0x43f4b6 0x437d57 0x46c309 0x4dd352 0x4de739 0x4de727 0x5d7749 0x5e78a5 0x4bb29a 0x68c179 0x68c143 0x745185 0x74514d 0x73cb89 0x73f8fd 0x73efac 0x7e03f6 0x7e352d 0x7e0095 0x7e001b 0x955822 0x8dc98a 0x955345 0x94afce 0x94bf6c 0x93cb4b 0x9c92e2 0x9c973a 0x471e01 # 0x46c308 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dd351 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4de738 internal/poll.(*pollDesc).waitRead+0x298 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4de726 internal/poll.(*FD).Read+0x286 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:167 # 0x5d7748 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e78a4 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x4bb299 io.ReadAtLeast+0x99 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:332 # 0x68c178 io.ReadFull+0xd8 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:351 # 0x68c142 github.com/couchbase/gomemcached.(*MCResponse).ReceiveWithBuf+0xa2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:169 # 0x745184 github.com/couchbase/gomemcached.(*MCResponse).Receive+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:156 # 0x74514c github.com/couchbase/gomemcached/client.getResponse+0x6c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/transport.go:35 # 0x73cb88 github.com/couchbase/gomemcached/client.(*Client).Send+0x68 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:280 # 0x73f8fc github.com/couchbase/gomemcached/client.(*Client).AuthPlain+0x23c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:759 # 0x73efab github.com/couchbase/gomemcached/client.(*Client).Auth+0xab /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:688 # 0x7e03f5 github.com/couchbase/goxdcr/base.authClient+0x75 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:236 # 0x7e352c github.com/couchbase/goxdcr/base.NewConn+0x1ec /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:612 # 0x7e0094 github.com/couchbase/goxdcr/base.(*connPool).newConn+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:208 # 0x7e001a github.com/couchbase/goxdcr/base.(*connPool).GetNew+0x1a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:204 # 0x955821 github.com/couchbase/goxdcr/parts.(*XmemNozzle).getClientWithRetry.func1+0x2a1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3774 # 0x8dc989 github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithFinishSignal+0x109 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2615 # 0x955344 github.com/couchbase/goxdcr/parts.(*XmemNozzle).getClientWithRetry+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3777 # 0x94afcd github.com/couchbase/goxdcr/parts.(*XmemNozzle).initializeConnection+0x38d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2406 # 0x94bf6b github.com/couchbase/goxdcr/parts.(*XmemNozzle).initialize+0x72b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2550 # 0x93cb4a github.com/couchbase/goxdcr/parts.(*XmemNozzle).Start+0x1ca /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:807 # 0x9c92e1 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart+0x321 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:233 # 0x9c9739 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart.func1+0xb9 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:209 2 @ 0x43f4b6 0x437d57 0x46c309 0x4dd352 0x4de739 0x4de727 0x5d7749 0x5e78a5 0x4bb29a 0x68c179 0x68c143 0x745185 0x74514d 0x73d0f3 0x747fdd 0x748e26 0x7483af 0x91db8b 0x91e6c7 0x91f78a 0x9c92e2 0x9c8d5f 0x471e01 # 0x46c308 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dd351 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4de738 internal/poll.(*pollDesc).waitRead+0x298 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4de726 internal/poll.(*FD).Read+0x286 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:167 # 0x5d7748 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e78a4 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x4bb299 io.ReadAtLeast+0x99 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:332 # 0x68c178 io.ReadFull+0xd8 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:351 # 0x68c142 github.com/couchbase/gomemcached.(*MCResponse).ReceiveWithBuf+0xa2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:169 # 0x745184 github.com/couchbase/gomemcached.(*MCResponse).Receive+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:156 # 0x74514c github.com/couchbase/gomemcached/client.getResponse+0x6c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/transport.go:35 # 0x73d0f2 github.com/couchbase/gomemcached/client.(*Client).Receive+0x52 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:333 # 0x747fdc github.com/couchbase/gomemcached/client.sendMcRequestSync+0x3c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/upr_feed.go:450 # 0x748e25 github.com/couchbase/gomemcached/client.(*UprFeed).uprOpen+0x765 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/upr_feed.go:592 # 0x7483ae github.com/couchbase/gomemcached/client.(*UprFeed).UprOpenWithFeatures+0x8e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/upr_feed.go:481 # 0x91db8a github.com/couchbase/goxdcr/parts.(*DcpNozzle).initializeUprFeed+0x2ca /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:575 # 0x91e6c6 github.com/couchbase/goxdcr/parts.(*DcpNozzle).initialize+0x5a6 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:739 # 0x91f789 github.com/couchbase/goxdcr/parts.(*DcpNozzle).Start+0x189 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:790 # 0x9c92e1 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart+0x321 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:233 # 0x9c8d5e github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPartsWithTimeout.func1.1+0xde /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:153 2 @ 0x43f4b6 0x44fa7e 0x721d46 0x71dc5e 0x705e59 0x6c6977 0x6c61fb 0x6c84fb 0x8d9413 0x8d940b 0x8d8693 0x8e9c62 0x8ed537 0x471e01 # 0x721d45 net/http.(*Transport).getConn+0x5c5 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:1382 # 0x71dc5d net/http.(*Transport).roundTrip+0x79d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:590 # 0x705e58 net/http.(*Transport).RoundTrip+0x18 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/roundtrip.go:17 # 0x6c6976 net/http.send+0x5f6 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:252 # 0x6c61fa net/http.(*Client).send+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:176 # 0x6c84fa net/http.(*Client).do+0x8fa /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:716 # 0x8d9412 net/http.(*Client).Do+0x52 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:582 # 0x8d940a github.com/couchbase/goxdcr/utils.(*Utilities).doRestCall+0x4a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2234 # 0x8d8692 github.com/couchbase/goxdcr/utils.(*Utilities).QueryRestApiWithAuth+0x1b2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2096 # 0x8e9c61 github.com/couchbase/goxdcr/peerToPeer.(*P2pCommAPIimpl).P2PSend+0x241 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerCommAPI.go:133 # 0x8ed536 github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).sendToSpecifiedPeersOnce.func1.1+0x316 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:309 2 @ 0x43f4b6 0x44fa7e 0x729d94 0x71dcba 0x705e59 0x6c6977 0x6c61fb 0x6c84fb 0x8d9413 0x8d940b 0x8d8693 0x8d2965 0x8ded31 0x955655 0x8dc98a 0x955345 0x94afce 0x94bf6c 0x93cb4b 0x9c92e2 0x9c973a 0x471e01 # 0x729d93 net/http.(*persistConn).roundTrip+0x993 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:2638 # 0x71dcb9 net/http.(*Transport).roundTrip+0x7f9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:603 # 0x705e58 net/http.(*Transport).RoundTrip+0x18 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/roundtrip.go:17 # 0x6c6976 net/http.send+0x5f6 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:252 # 0x6c61fa net/http.(*Client).send+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:176 # 0x6c84fa net/http.(*Client).do+0x8fa /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:716 # 0x8d9412 net/http.(*Client).Do+0x52 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:582 # 0x8d940a github.com/couchbase/goxdcr/utils.(*Utilities).doRestCall+0x4a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2234 # 0x8d8692 github.com/couchbase/goxdcr/utils.(*Utilities).QueryRestApiWithAuth+0x1b2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2096 # 0x8d2964 github.com/couchbase/goxdcr/utils.(*Utilities).GetBucketInfo+0x324 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:1232 # 0x8ded30 github.com/couchbase/goxdcr/utils.(*Utilities).VerifyTargetBucket+0x170 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2844 # 0x955654 github.com/couchbase/goxdcr/parts.(*XmemNozzle).getClientWithRetry.func1+0xd4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3752 # 0x8dc989 github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithFinishSignal+0x109 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2615 # 0x955344 github.com/couchbase/goxdcr/parts.(*XmemNozzle).getClientWithRetry+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3777 # 0x94afcd github.com/couchbase/goxdcr/parts.(*XmemNozzle).initializeConnection+0x38d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2406 # 0x94bf6b github.com/couchbase/goxdcr/parts.(*XmemNozzle).initialize+0x72b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2550 # 0x93cb4a github.com/couchbase/goxdcr/parts.(*XmemNozzle).Start+0x1ca /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:807 # 0x9c92e1 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart+0x321 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:233 # 0x9c9739 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart.func1+0xb9 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:209 2 @ 0x43f4b6 0x44fa7e 0x729d94 0x71dcba 0x705e59 0x8c67ec 0x6c6977 0x6c61fb 0x6c84fb 0x95c9a5 0x95c991 0x95cb65 0x95cdad 0x95cee9 0x984186 0x984187 0x8dc6bb 0x8dc47b 0x983ed2 0x96d110 0x96eed6 0x9f41ce 0x9f182f 0x9f2a38 0x9f0f59 0x9effe6 0x471e01 # 0x729d93 net/http.(*persistConn).roundTrip+0x993 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:2638 # 0x71dcb9 net/http.(*Transport).roundTrip+0x7f9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:603 # 0x705e58 net/http.(*Transport).RoundTrip+0x18 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/roundtrip.go:17 # 0x8c67eb github.com/couchbase/cbauth.(*cbauthRoundTripper).RoundTrip+0x46b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/convenience.go:76 # 0x6c6976 net/http.send+0x5f6 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:252 # 0x6c61fa net/http.(*Client).send+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:176 # 0x6c84fa net/http.(*Client).do+0x8fa /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:716 # 0x95c9a4 net/http.(*Client).Do+0x324 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:582 # 0x95c990 github.com/couchbase/cbauth/metakv.doCallInner+0x310 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:102 # 0x95cb64 github.com/couchbase/cbauth/metakv.doCall+0x64 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:119 # 0x95cdac github.com/couchbase/cbauth/metakv.doJSONCall+0x2c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:130 # 0x95cee8 github.com/couchbase/cbauth/metakv.(*store).get+0xa8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:171 # 0x984185 github.com/couchbase/cbauth/metakv.Get+0x85 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:333 # 0x984186 github.com/couchbase/goxdcr/metadata_svc.(*MetaKVMetadataSvc).Get.func1+0x86 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/metakv_metadata_service.go:50 # 0x8dc6ba github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithOriginalError+0xba /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2587 # 0x8dc47a github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutor+0x3a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2575 # 0x983ed1 github.com/couchbase/goxdcr/metadata_svc.(*MetaKVMetadataSvc).Get+0x251 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/metakv_metadata_service.go:63 # 0x96d10f github.com/couchbase/goxdcr/metadata_svc.(*CheckpointsService).checkpointsDocInternal+0x20f /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/checkpoints_service.go:120 # 0x96eed5 github.com/couchbase/goxdcr/metadata_svc.(*CheckpointsService).UpsertCheckpoints+0x315 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/checkpoints_service.go:343 # 0x9f41cd github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).persistCkptRecord+0x20d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:2150 # 0x9f182e github.com/couchbase/goxdcr/pipeline_svc.(*checkpointRecordWithLock).updateAndPersist+0x12e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1944 # 0x9f2a37 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).doCheckpoint+0xd37 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:2067 # 0x9f0f58 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).performCkpt_internal+0x738 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1894 # 0x9effe5 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).performCkpt+0x7c5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1828 2 @ 0x43f4b6 0x44fa7e 0x729d94 0x71dcba 0x705e59 0x8c67ec 0x6c6977 0x6c61fb 0x6c84fb 0x95c9a5 0x95c991 0x95cb65 0x95d43e 0x95d5d4 0x985cae 0x985c7a 0x8dc6bb 0x8dc47b 0x985942 0x98542a 0x96fb03 0x97529c 0x9ff582 0x9fe6fc 0x471e01 # 0x729d93 net/http.(*persistConn).roundTrip+0x993 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:2638 # 0x71dcb9 net/http.(*Transport).roundTrip+0x7f9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:603 # 0x705e58 net/http.(*Transport).RoundTrip+0x18 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/roundtrip.go:17 # 0x8c67eb github.com/couchbase/cbauth.(*cbauthRoundTripper).RoundTrip+0x46b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/convenience.go:76 # 0x6c6976 net/http.send+0x5f6 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:252 # 0x6c61fa net/http.(*Client).send+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:176 # 0x6c84fa net/http.(*Client).do+0x8fa /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:716 # 0x95c9a4 net/http.(*Client).Do+0x324 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:582 # 0x95c990 github.com/couchbase/cbauth/metakv.doCallInner+0x310 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:102 # 0x95cb64 github.com/couchbase/cbauth/metakv.doCall+0x64 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:119 # 0x95d43d github.com/couchbase/cbauth/metakv.mutate+0x41d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:209 # 0x95d5d3 github.com/couchbase/cbauth/metakv.(*store).set+0x113 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:219 # 0x985cad github.com/couchbase/cbauth/metakv.Set+0x14d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:341 # 0x985c79 github.com/couchbase/goxdcr/metadata_svc.(*MetaKVMetadataSvc).set.func2+0x119 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/metakv_metadata_service.go:201 # 0x8dc6ba github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithOriginalError+0xba /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2587 # 0x8dc47a github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutor+0x3a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2575 # 0x985941 github.com/couchbase/goxdcr/metadata_svc.(*MetaKVMetadataSvc).set+0x3e1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/metakv_metadata_service.go:215 # 0x985429 github.com/couchbase/goxdcr/metadata_svc.(*MetaKVMetadataSvc).Set+0x49 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/metakv_metadata_service.go:165 # 0x96fb02 github.com/couchbase/goxdcr/metadata_svc.(*CheckpointsService).upsertCheckpointsDoc+0x3c2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/checkpoints_service.go:414 # 0x97529b github.com/couchbase/goxdcr/metadata_svc.(*CheckpointsService).UpsertAndReloadCheckpointCompleteSet+0x27b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/checkpoints_service.go:1046 # 0x9ff581 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).mergeAndPersistBrokenMappingDocsAndCkpts+0x3e1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:3317 # 0x9fe6fb github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).mergeFinalCkpts.func1+0x39b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:3213 2 @ 0x43f4b6 0x44fa7e 0x7e727f 0x9c821d 0x9ca753 0xa2f318 0xa39525 0xa389fb 0xa37bc6 0x48d322 0xa373c7 0xa37395 0x471e01 # 0x7e727e github.com/couchbase/goxdcr/base.ExecWithTimeout+0x19e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/simple_utils.go:56 # 0x9c821c github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPartsWithTimeout+0x13c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:175 # 0x9ca752 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).Start+0xf32 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:326 # 0xa2f317 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineManager).StartPipeline+0x4d7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:464 # 0xa39524 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).update+0x424 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1899 # 0xa389fa github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).checkIfNeedToReUpdate+0x11a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1739 # 0xa37bc5 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run.func1+0x7c5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1621 # 0x48d321 sync.(*Once).doSlow+0xc1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:74 # 0xa373c6 sync.(*Once).Do+0x46 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:65 # 0xa37394 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run+0x14 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1588 2 @ 0x43f4b6 0x450b6f 0x450b46 0x46db27 0x48ea8b 0x8ed046 0x8dc6bb 0x8dc47b 0x8ec569 0x8f0174 0x471e01 # 0x46db26 sync.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:62 # 0x48ea8a sync.(*WaitGroup).Wait+0x4a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/waitgroup.go:116 # 0x8ed045 github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).sendToSpecifiedPeersOnce.func1+0x3e5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:344 # 0x8dc6ba github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithOriginalError+0xba /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2587 # 0x8dc47a github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutor+0x3a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2575 # 0x8ec568 github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).sendToSpecifiedPeersOnce+0x328 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:356 # 0x8f0173 github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).sendPeriodicPushRequest.func1+0x2b3 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:682 2 @ 0x43f4b6 0x450b6f 0x450b46 0x46db27 0x48ea8b 0x9e5965 0xa6234e 0x471e01 # 0x46db26 sync.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:62 # 0x48ea8a sync.(*WaitGroup).Wait+0x4a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/waitgroup.go:116 # 0x9e5964 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).Stop+0x44 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:843 # 0xa6234d github.com/couchbase/goxdcr/pipeline_ctx.(*PipelineRuntimeCtx).stopService+0xed /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_ctx/pipeline_runtimeCtx.go:201 2 @ 0x43f4b6 0x450b6f 0x450b46 0x46db27 0x48ea8b 0x9fe0fe 0x9fa8fd 0xa016d5 0x471e01 # 0x46db26 sync.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:62 # 0x48ea8a sync.(*WaitGroup).Wait+0x4a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/waitgroup.go:116 # 0x9fe0fd github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).mergeFinalCkpts+0x41d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:3220 # 0x9fa8fc github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).stopTheWorldAndMergeCkpts+0x23c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:2764 # 0xa016d4 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).periodicMergerImpl+0x194 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:3542 2 @ 0x43f4b6 0x450b6f 0x450b46 0x46db27 0x48ea8b 0xa61b6f 0x7e7482 0x471e01 # 0x46db26 sync.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:62 # 0x48ea8a sync.(*WaitGroup).Wait+0x4a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/waitgroup.go:116 # 0xa61b6e github.com/couchbase/goxdcr/pipeline_ctx.(*PipelineRuntimeCtx).Stop.func2+0x38e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_ctx/pipeline_runtimeCtx.go:170 # 0x7e7481 github.com/couchbase/goxdcr/base.ExecWithTimeout.func1+0x21 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/simple_utils.go:48 2 @ 0x43f4b6 0x450b6f 0x450b46 0x46db27 0x48ea8b 0xda2bb7 0xda0ca5 0x471e01 # 0x46db26 sync.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:62 # 0x48ea8a sync.(*WaitGroup).Wait+0x4a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/waitgroup.go:116 # 0xda2bb6 github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).updateOnce+0x9b6 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1411 # 0xda0ca4 github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).run.func2+0x1e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1176 2 @ 0x43f4b6 0x450b6f 0x450b46 0x46db87 0x4dcdd2 0x4e06f2 0x4e0703 0x4e9465 0x4e945b 0x5118e6 0x50ef6a 0x50f0dd 0x5121fc 0xa11c73 0xa11b9e 0xa117ea 0xa1158f 0xa10933 0xa1010f 0xa0f225 0xa0e745 0x471e01 # 0x46db86 internal/poll.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:67 # 0x4dcdd1 internal/poll.(*fdMutex).rwlock+0xd1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:154 # 0x4e06f1 internal/poll.(*FD).writeLock+0x71 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:239 # 0x4e0702 internal/poll.(*FD).Write+0x82 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:370 # 0x4e9464 os.(*File).write+0x64 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file_posix.go:48 # 0x4e945a os.(*File).Write+0x5a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file.go:175 # 0x5118e5 github.com/couchbase/goxdcr/log.(*LogWriter).Write+0x25 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:94 # 0x50ef69 log.(*Logger).Output+0x349 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:194 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0xa11c72 github.com/couchbase/goxdcr/log.(*CommonLogger).Infof+0x112 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:193 # 0xa11b9d github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).logChangesLeft+0x3d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:699 # 0xa117e9 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).calculateChangesLeftAndDocsProcessedMainPipeline+0x229 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:684 # 0xa1158e github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).calculateChangesLeftAndDocProcessed+0x10e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:654 # 0xa10932 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).processCalculatedStats+0x72 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:548 # 0xa1010e github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).processRawStats+0x7ee /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:528 # 0xa0f224 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).updateStatsOnce+0x144 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:355 # 0xa0e744 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).updateStats+0x3c4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:303 2 @ 0x43f4b6 0x450b6f 0x450b46 0x46db87 0x4dcdd2 0x4e06f2 0x4e0703 0x4e9465 0x4e945b 0x5118e6 0x50ef6a 0x50f23a 0x512329 0xa0f3ee 0xa0f3d0 0xa0ee3b 0x471e01 # 0x46db86 internal/poll.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:67 # 0x4dcdd1 internal/poll.(*fdMutex).rwlock+0xd1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:154 # 0x4e06f1 internal/poll.(*FD).writeLock+0x71 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:239 # 0x4e0702 internal/poll.(*FD).Write+0x82 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:370 # 0x4e9464 os.(*File).write+0x64 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file_posix.go:48 # 0x4e945a os.(*File).Write+0x5a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file.go:175 # 0x5118e5 github.com/couchbase/goxdcr/log.(*LogWriter).Write+0x25 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:94 # 0x50ef69 log.(*Logger).Output+0x349 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:194 # 0x50f239 log.(*Logger).Println+0x59 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:222 # 0x512328 github.com/couchbase/goxdcr/log.(*CommonLogger).logMsg+0xc8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:176 # 0xa0f3ed github.com/couchbase/goxdcr/log.(*CommonLogger).Info+0x6d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:217 # 0xa0f3cf github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).logStatsOnce+0x4f /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:369 # 0xa0ee3a github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).logStats+0x23a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:336 2 @ 0x43f4b6 0x450b6f 0x450b46 0x46dc46 0x48cf85 0x48e5d6 0x48e5b5 0x68e6ce 0x73f7cd 0x73efac 0x7e03f6 0x7e352d 0x7e0095 0x7e001b 0x955822 0x8dc98a 0x955345 0x94b06f 0x94bf6c 0x93cb4b 0x9c92e2 0x9c973a 0x471e01 # 0x46dc45 sync.runtime_SemacquireMutex+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:77 # 0x48cf84 sync.(*Mutex).lockSlow+0x164 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:171 # 0x48e5d5 sync.(*Mutex).Lock+0x35 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:90 # 0x48e5b4 sync.(*RWMutex).Lock+0x14 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/rwmutex.go:147 # 0x68e6cd github.com/couchbase/goutils/logging.Infof+0x6d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goutils/logging/logger.go:289 # 0x73f7cc github.com/couchbase/gomemcached/client.(*Client).AuthPlain+0x10c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:757 # 0x73efab github.com/couchbase/gomemcached/client.(*Client).Auth+0xab /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:688 # 0x7e03f5 github.com/couchbase/goxdcr/base.authClient+0x75 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:236 # 0x7e352c github.com/couchbase/goxdcr/base.NewConn+0x1ec /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:612 # 0x7e0094 github.com/couchbase/goxdcr/base.(*connPool).newConn+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:208 # 0x7e001a github.com/couchbase/goxdcr/base.(*connPool).GetNew+0x1a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:204 # 0x955821 github.com/couchbase/goxdcr/parts.(*XmemNozzle).getClientWithRetry.func1+0x2a1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3774 # 0x8dc989 github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithFinishSignal+0x109 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2615 # 0x955344 github.com/couchbase/goxdcr/parts.(*XmemNozzle).getClientWithRetry+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3777 # 0x94b06e github.com/couchbase/goxdcr/parts.(*XmemNozzle).initializeConnection+0x42e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2415 # 0x94bf6b github.com/couchbase/goxdcr/parts.(*XmemNozzle).initialize+0x72b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2550 # 0x93cb4a github.com/couchbase/goxdcr/parts.(*XmemNozzle).Start+0x1ca /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:807 # 0x9c92e1 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart+0x321 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:233 # 0x9c9739 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart.func1+0xb9 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:209 2 @ 0x43f4b6 0x450b6f 0x450b46 0x46dca6 0x939d9f 0x939d7b 0x94dfdd 0x471e01 # 0x46dca5 sync.runtime_SemacquireRWMutexR+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:82 # 0x939d9e sync.(*RWMutex).RLock+0x9e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/rwmutex.go:71 # 0x939d7a github.com/couchbase/goxdcr/parts.(*requestBuffer).slotWithSentTime+0x7a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:280 # 0x94dfdc github.com/couchbase/goxdcr/parts.(*XmemNozzle).receiveResponse+0x177c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2758 2 @ 0x43f4b6 0x46ebf5 0x8dc7c8 0x8dc47b 0x8ec569 0x8ec1dd 0x8eee2d 0xdd1e51 0x9cb509 0x9c9f8d 0xa2f318 0xa39525 0xa389fb 0xa37bc6 0x48d322 0xa373c7 0xa37395 0x471e01 # 0x46ebf4 time.Sleep+0x134 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/time.go:195 # 0x8dc7c7 github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithOriginalError+0x1c7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2593 # 0x8dc47a github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutor+0x3a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2575 # 0x8ec568 github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).sendToSpecifiedPeersOnce+0x328 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:356 # 0x8ec1dc github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).sendToEachPeerOnce+0xfc /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:257 # 0x8eee2c github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).CheckVBMaster+0x30c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:541 # 0xdd1e50 github.com/couchbase/goxdcr/factory.(*XDCRFactory).PreReplicationVBMasterCheck+0x2b0 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/factory/xdcr_factory.go:933 # 0x9cb508 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).runP2PProtocol+0xe8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:376 # 0x9c9f8c github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).Start+0x76c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:274 # 0xa2f317 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineManager).StartPipeline+0x4d7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:464 # 0xa39524 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).update+0x424 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1899 # 0xa389fa github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).checkIfNeedToReUpdate+0x11a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1739 # 0xa37bc5 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run.func1+0x7c5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1621 # 0x48d321 sync.(*Once).doSlow+0xc1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:74 # 0xa373c6 sync.(*Once).Do+0x46 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:65 # 0xa37394 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run+0x14 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1588 2 @ 0x4ceec5 0x8000c7 0x7ffe69 0x9522bb 0x95335d 0x94c9c5 0x471e01 # 0x4ceec4 time.Time.Add+0x144 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/time/time.go:856 # 0x8000c6 github.com/couchbase/goxdcr/base.(*XmemClient).setReadTimeout+0x46 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/xmem_client.go:156 # 0x7ffe68 github.com/couchbase/goxdcr/base.(*XmemClient).GetConn+0x108 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/xmem_client.go:145 # 0x9522ba github.com/couchbase/goxdcr/parts.(*XmemNozzle).getConn+0x9a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3298 # 0x95335c github.com/couchbase/goxdcr/parts.(*XmemNozzle).readFromClient+0x3c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3488 # 0x94c9c4 github.com/couchbase/goxdcr/parts.(*XmemNozzle).receiveResponse+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2618 2 @ 0x4cf0aa 0x4cf154 0x95088c 0x939f54 0x9506c9 0x471e01 # 0x4cf0a9 time.Time.Sub+0x1c9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/time/time.go:884 # 0x4cf153 time.Since+0x73 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/time/time.go:919 # 0x95088b github.com/couchbase/goxdcr/parts.(*XmemNozzle).resendIfTimeout+0xab /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3041 # 0x939f53 github.com/couchbase/goxdcr/parts.(*requestBuffer).modSlot+0x53 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:296 # 0x9506c8 github.com/couchbase/goxdcr/parts.(*XmemNozzle).checkAndRepairBufferMonitor+0x348 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2995 2 @ 0x4de811 0x5d7749 0x5e78a5 0x4bb29a 0x68b4dc 0x68b4ae 0x74ab8d 0x471e01 # 0x4de810 internal/poll.(*FD).Read+0x370 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:173 # 0x5d7748 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e78a4 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x4bb299 io.ReadAtLeast+0x99 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:332 # 0x68b4db io.ReadFull+0x9b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:351 # 0x68b4ad github.com/couchbase/gomemcached.(*MCRequest).Receive+0x6d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_req.go:643 # 0x74ab8c github.com/couchbase/gomemcached/client.(*UprFeed).runFeed+0x14c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/upr_feed.go:904 2 @ 0x68c610 0x745185 0x74514d 0x743176 0x8e05f5 0xd9e5e3 0xda2771 0xda0ca5 0x471e01 # 0x68c60f github.com/couchbase/gomemcached.(*MCResponse).ReceiveWithBuf+0x56f /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:162 # 0x745184 github.com/couchbase/gomemcached.(*MCResponse).Receive+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:156 # 0x74514c github.com/couchbase/gomemcached/client.getResponse+0x6c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/transport.go:35 # 0x743175 github.com/couchbase/gomemcached/client.(*Client).StatsMapForSpecifiedStats+0x1b5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:1461 # 0x8e05f4 github.com/couchbase/goxdcr/utils.(*Utilities).GetHighSeqNos+0x154 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:3016 # 0xd9e5e2 github.com/couchbase/goxdcr/service_impl.(*BucketTopologyService).getHighSeqnosUpdater.func1+0x942 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:814 # 0xda2770 github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).updateOnce+0x570 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1362 # 0xda0ca4 github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).run.func2+0x1e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1176 2 @ 0x74c1f4 0x9213f7 0x471e01 # 0x74c1f3 github.com/couchbase/gomemcached/client.(*UprFeed).ClientAck+0x73 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/upr_feed.go:1070 # 0x9213f6 github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0x316 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1006 2 @ 0x90e085 0x94e93e 0x471e01 # 0x90e084 github.com/couchbase/goxdcr/component.(*AbstractComponent).RaiseEvent+0x124 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/component/abstract_component.go:85 # 0x94e93d github.com/couchbase/goxdcr/parts.(*XmemNozzle).receiveResponse+0x20dd /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2791 2 @ 0x91025b 0x921d87 0x471e01 # 0x91025a github.com/couchbase/goxdcr/parts.(*AbstractPart).Connector+0xda /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/abstract_part.go:50 # 0x921d86 github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0xca6 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1102 1 @ 0x4081f7 0x4147fe 0x7454f3 0x74b2e5 0x471e01 # 0x7454f2 github.com/couchbase/gomemcached/client.makeUprEvent+0x192 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/upr_event.go:118 # 0x74b2e4 github.com/couchbase/gomemcached/client.(*UprFeed).runFeed+0x8a4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/upr_feed.go:945 1 @ 0x408257 0x4145be 0x74acf5 0x471e01 # 0x74acf4 github.com/couchbase/gomemcached/client.(*UprFeed).runFeed+0x2b4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/upr_feed.go:928 1 @ 0x40985c 0xa441a5 0xa44e93 0xa44d25 0xa44b8a 0xa404f7 0xa3fa53 0x471e01 # 0xa441a4 github.com/couchbase/goxdcr/resource_manager._Cfunc_sigar_get_control_group_info+0x44 _cgo_gotypes.go:148 # 0xa44e92 github.com/couchbase/goxdcr/resource_manager.(*SystemStats).GetControlGroupInfo+0x32 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/resource_manager/system_stats.go:166 # 0xa44d24 github.com/couchbase/goxdcr/resource_manager.(*SystemStats).overallCpuCgroup+0x24 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/resource_manager/system_stats.go:125 # 0xa44b89 github.com/couchbase/goxdcr/resource_manager.(*SystemStats).OverallCpu+0x109 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/resource_manager/system_stats.go:111 # 0xa404f6 github.com/couchbase/goxdcr/resource_manager.(*ResourceManager).collectCpuUsageOnce+0x136 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/resource_manager/resource_manager.go:577 # 0xa3fa52 github.com/couchbase/goxdcr/resource_manager.(*ResourceManager).collectCpuUsage+0x112 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/resource_manager/resource_manager.go:500 1 @ 0x40ba91 0x40b43d 0x93a73e 0x940030 0x941648 0x93f449 0x93f025 0x471e01 # 0x93a73d github.com/couchbase/goxdcr/parts.(*requestBuffer).enSlot+0x2dd /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:390 # 0x94002f github.com/couchbase/goxdcr/parts.(*XmemNozzle).batchSetMetaWithRetry+0x6ef /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1202 # 0x941647 github.com/couchbase/goxdcr/parts.(*XmemNozzle).sendSetMeta_internal+0x47 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1448 # 0x93f448 github.com/couchbase/goxdcr/parts.(*XmemNozzle).processBatch+0x48 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1121 # 0x93f024 github.com/couchbase/goxdcr/parts.(*XmemNozzle).processData_sendbatch+0x4e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1077 1 @ 0x40ba91 0x40b43d 0x94c969 0x471e01 # 0x94c968 github.com/couchbase/goxdcr/parts.(*XmemNozzle).receiveResponse+0x108 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2612 1 @ 0x40cb9e 0x954b4c 0x954b1c 0x93fa2c 0x941648 0x93f449 0x93f025 0x471e01 # 0x954b4b github.com/couchbase/goxdcr/parts.(*XmemNozzle).dataChanControl+0x18b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3654 # 0x954b1b github.com/couchbase/goxdcr/parts.(*XmemNozzle).readFromDataChan+0x15b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3685 # 0x93fa2b github.com/couchbase/goxdcr/parts.(*XmemNozzle).batchSetMetaWithRetry+0xeb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1180 # 0x941647 github.com/couchbase/goxdcr/parts.(*XmemNozzle).sendSetMeta_internal+0x47 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1448 # 0x93f448 github.com/couchbase/goxdcr/parts.(*XmemNozzle).processBatch+0x48 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1121 # 0x93f024 github.com/couchbase/goxdcr/parts.(*XmemNozzle).processData_sendbatch+0x4e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1077 1 @ 0x410e69 0x90e6d6 0x9ca207 0xa2f318 0xa39525 0xa389fb 0xa37bc6 0x48d322 0xa373c7 0xa37395 0x471e01 # 0x90e6d5 github.com/couchbase/goxdcr/component.(*AsyncComponentEventListenerImpl).Start+0x95 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/component/async_listener.go:61 # 0x9ca206 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).Start+0x9e6 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:287 # 0xa2f317 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineManager).StartPipeline+0x4d7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:464 # 0xa39524 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).update+0x424 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1899 # 0xa389fa github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).checkIfNeedToReUpdate+0x11a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1739 # 0xa37bc5 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run.func1+0x7c5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1621 # 0x48d321 sync.(*Once).doSlow+0xc1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:74 # 0xa373c6 sync.(*Once).Do+0x46 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:65 # 0xa37394 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run+0x14 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1588 1 @ 0x410f15 0x7e6a5f 0x48d5d9 0x7e6af2 0x9376b7 0x93723a 0x936649 0x90f4c2 0x921da2 0x471e01 # 0x7e6a5e github.com/couchbase/goxdcr/base.NewDataPool.func1+0x3e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/datapool.go:75 # 0x48d5d8 sync.(*Pool).Get+0xb8 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/pool.go:151 # 0x7e6af1 github.com/couchbase/goxdcr/base.(*DataPoolImpl).GetByteSlice+0x71 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/datapool.go:106 # 0x9376b6 github.com/couchbase/goxdcr/parts.(*Router).prepareMcRequest+0x356 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/router.go:1795 # 0x937239 github.com/couchbase/goxdcr/parts.(*Router).RouteCollection+0x4d9 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/router.go:1748 # 0x936648 github.com/couchbase/goxdcr/parts.(*Router).Route+0x648 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/router.go:1679 # 0x90f4c1 github.com/couchbase/goxdcr/connector.(*Router).Forward+0x141 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/connector/router.go:113 # 0x921da1 github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0xcc1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1102 1 @ 0x41105f 0x8000e7 0x7ffe69 0x9522bb 0x95335d 0x94c9c5 0x471e01 # 0x8000e6 github.com/couchbase/goxdcr/base.(*XmemClient).setReadTimeout+0x66 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/xmem_client.go:156 # 0x7ffe68 github.com/couchbase/goxdcr/base.(*XmemClient).GetConn+0x108 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/xmem_client.go:145 # 0x9522ba github.com/couchbase/goxdcr/parts.(*XmemNozzle).getConn+0x9a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3298 # 0x95335c github.com/couchbase/goxdcr/parts.(*XmemNozzle).readFromClient+0x3c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3488 # 0x94c9c4 github.com/couchbase/goxdcr/parts.(*XmemNozzle).receiveResponse+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2618 1 @ 0x4111c7 0xa18985 0x90e8f8 0x471e01 # 0xa18984 github.com/couchbase/goxdcr/pipeline_svc.(*outNozzleCollector).ProcessEvent+0x364 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:1372 # 0x90e8f7 github.com/couchbase/goxdcr/component.(*AsyncComponentEventListenerImpl).run+0x157 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/component/async_listener.go:74 1 @ 0x41359b 0x921e4b 0x921e22 0x471e01 # 0x921e4a github.com/couchbase/goxdcr/common.NewEvent+0xd6a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/common/component.go:118 # 0x921e21 github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0xd41 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1110 1 @ 0x41359b 0x938c36 0x9352bb 0x9363e5 0x90f4c2 0x921da2 0x471e01 # 0x938c35 github.com/couchbase/goxdcr/parts.(*Router).newWrappedMCRequest+0x35 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/router.go:1974 # 0x9352ba github.com/couchbase/goxdcr/parts.(*Router).ComposeMCRequest+0x3a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/router.go:1460 # 0x9363e4 github.com/couchbase/goxdcr/parts.(*Router).Route+0x3e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/router.go:1663 # 0x90f4c1 github.com/couchbase/goxdcr/connector.(*Router).Forward+0x141 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/connector/router.go:113 # 0x921da1 github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0xcc1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1102 1 @ 0x414750 0x7fd9e5 0xda6805 0xa1460e 0x471e01 # 0x7fd9e4 github.com/couchbase/goxdcr/base.(*VbHostsMapType).GreenClone+0x2c4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/types.go:2785 # 0xda6804 github.com/couchbase/goxdcr/service_impl.(*Notification).Clone+0x344 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1938 # 0xa1460d github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).watchNotificationCh+0x1cd /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:1050 1 @ 0x4150f2 0x8c880f 0x7fd3d5 0xda6785 0x8fdde6 0x471e01 # 0x8c880e github.com/couchbase/goxdcr/utils.(*VbSeqnoMapPool).Get+0xee /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/dataObjPools.go:295 # 0x7fd3d4 github.com/couchbase/goxdcr/base.(*HighSeqnosMapType).GreenClone+0x354 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/types.go:2745 # 0xda6784 github.com/couchbase/goxdcr/service_impl.(*Notification).Clone+0x2c4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1936 # 0x8fdde5 github.com/couchbase/goxdcr/peerToPeer.(*ReplicaCacheMonitor).Run+0x105 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/replicaCache.go:121 1 @ 0x415ca5 0x415756 0x90f587 0x921da2 0x471e01 # 0x90f586 github.com/couchbase/goxdcr/connector.(*Router).Forward+0x206 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/connector/router.go:115 # 0x921da1 github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0xcc1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1102 1 @ 0x415ca5 0x7fd8be 0xda6805 0x8fdde6 0x471e01 # 0x7fd8bd github.com/couchbase/goxdcr/base.(*VbHostsMapType).GreenClone+0x19d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/types.go:2780 # 0xda6804 github.com/couchbase/goxdcr/service_impl.(*Notification).Clone+0x344 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1938 # 0x8fdde5 github.com/couchbase/goxdcr/peerToPeer.(*ReplicaCacheMonitor).Run+0x105 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/replicaCache.go:121 1 @ 0x415ca5 0x8c8a3a 0x7fd85f 0xda6805 0xa57678 0x471e01 # 0x8c8a39 github.com/couchbase/goxdcr/utils.(*VbHostsMapPool).Get+0x139 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/dataObjPools.go:374 # 0x7fd85e github.com/couchbase/goxdcr/base.(*VbHostsMapType).GreenClone+0x13e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/types.go:2779 # 0xda6804 github.com/couchbase/goxdcr/service_impl.(*Notification).Clone+0x344 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1938 # 0xa57677 github.com/couchbase/goxdcr/backfill_manager.(*BackfillRequestHandler).run+0x13d7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/backfill_manager/backfill_request_handler.go:337 1 @ 0x41f13b 0x413308 0x413587 0x745147 0x743176 0x8e05f5 0xd9e5e3 0xda2771 0xda0ca5 0x471e01 # 0x745146 github.com/couchbase/gomemcached/client.getResponse+0x66 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/transport.go:34 # 0x743175 github.com/couchbase/gomemcached/client.(*Client).StatsMapForSpecifiedStats+0x1b5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:1461 # 0x8e05f4 github.com/couchbase/goxdcr/utils.(*Utilities).GetHighSeqNos+0x154 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:3016 # 0xd9e5e2 github.com/couchbase/goxdcr/service_impl.(*BucketTopologyService).getHighSeqnosUpdater.func1+0x942 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:814 # 0xda2770 github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).updateOnce+0x570 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1362 # 0xda0ca4 github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).run.func2+0x1e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1176 1 @ 0x41f13b 0x413308 0x413587 0x805b09 0x48d5d9 0x7e6d1d 0x938c25 0x9352bb 0x9363e5 0x90f4c2 0x921da2 0x471e01 # 0x805b08 github.com/couchbase/goxdcr/base.(*MCRequestPool).addOne+0x28 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/mcreqeust_pool.go:47 # 0x48d5d8 sync.(*Pool).Get+0xb8 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/pool.go:151 # 0x7e6d1c github.com/couchbase/goxdcr/base.(*MCRequestPool).Get+0x1c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/mcreqeust_pool.go:37 # 0x938c24 github.com/couchbase/goxdcr/parts.(*Router).newWrappedMCRequest+0x24 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/router.go:1973 # 0x9352ba github.com/couchbase/goxdcr/parts.(*Router).ComposeMCRequest+0x3a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/router.go:1460 # 0x9363e4 github.com/couchbase/goxdcr/parts.(*Router).Route+0x3e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/router.go:1663 # 0x90f4c1 github.com/couchbase/goxdcr/connector.(*Router).Forward+0x141 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/connector/router.go:113 # 0x921da1 github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0xcc1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1102 1 @ 0x434816 0x46bee5 0x7d1bf5 0x7d1a0d 0x7ce78b 0x7dda45 0x7de593 0x70ecef 0x710689 0x711e36 0x70d812 0x471e01 # 0x46bee4 runtime/pprof.runtime_goroutineProfileWithLabels+0x24 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/mprof.go:844 # 0x7d1bf4 runtime/pprof.writeRuntimeProfile+0xb4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/pprof/pprof.go:734 # 0x7d1a0c runtime/pprof.writeGoroutine+0x4c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/pprof/pprof.go:694 # 0x7ce78a runtime/pprof.(*Profile).WriteTo+0x14a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/pprof/pprof.go:329 # 0x7dda44 net/http/pprof.handler.ServeHTTP+0x4a4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/pprof/pprof.go:259 # 0x7de592 net/http/pprof.Index+0xf2 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/pprof/pprof.go:376 # 0x70ecee net/http.HandlerFunc.ServeHTTP+0x2e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/server.go:2122 # 0x710688 net/http.(*ServeMux).ServeHTTP+0x148 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/server.go:2500 # 0x711e35 net/http.serverHandler.ServeHTTP+0x315 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/server.go:2936 # 0x70d811 net/http.(*conn).serve+0x611 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/server.go:1995 1 @ 0x43dbbf 0x47345b 0x48e747 0x93a7a6 0x940030 0x941648 0x93f449 0x93f025 0x471e01 # 0x48e746 sync.(*RWMutex).Unlock+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/rwmutex.go:206 # 0x93a7a5 github.com/couchbase/goxdcr/parts.(*requestBuffer).enSlot+0x345 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:395 # 0x94002f github.com/couchbase/goxdcr/parts.(*XmemNozzle).batchSetMetaWithRetry+0x6ef /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1202 # 0x941647 github.com/couchbase/goxdcr/parts.(*XmemNozzle).sendSetMeta_internal+0x47 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1448 # 0x93f448 github.com/couchbase/goxdcr/parts.(*XmemNozzle).processBatch+0x48 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1121 # 0x93f024 github.com/couchbase/goxdcr/parts.(*XmemNozzle).processData_sendbatch+0x4e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1077 1 @ 0x43dbbf 0x47345b 0x48e79d 0x938fd4 0x954cb6 0x94e98c 0x471e01 # 0x48e79c sync.(*RWMutex).Unlock+0x7c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/rwmutex.go:212 # 0x938fd3 github.com/couchbase/goxdcr/parts.(*Router).recycleDataObj+0x153 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/router.go:2036 # 0x954cb5 github.com/couchbase/goxdcr/parts.(*XmemNozzle).recycleDataObj+0xb5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3713 # 0x94e98b github.com/couchbase/goxdcr/parts.(*XmemNozzle).receiveResponse+0x212b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2802 1 @ 0x43dbbf 0x47345b 0x553a8b 0x5538e9 0x5538df 0x5538d5 0x93a5c5 0x940030 0x941648 0x93f449 0x93f025 0x471e01 # 0x553a8a math/rand.(*lockedSource).Int63+0x2a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/math/rand/rand.go:429 # 0x5538e8 math/rand.(*Rand).Int63+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/math/rand/rand.go:95 # 0x5538de math/rand.(*Rand).Int+0x1e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/math/rand/rand.go:113 # 0x5538d4 math/rand.Int+0x14 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/math/rand/rand.go:343 # 0x93a5c4 github.com/couchbase/goxdcr/parts.(*requestBuffer).enSlot+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:374 # 0x94002f github.com/couchbase/goxdcr/parts.(*XmemNozzle).batchSetMetaWithRetry+0x6ef /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1202 # 0x941647 github.com/couchbase/goxdcr/parts.(*XmemNozzle).sendSetMeta_internal+0x47 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1448 # 0x93f448 github.com/couchbase/goxdcr/parts.(*XmemNozzle).processBatch+0x48 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1121 # 0x93f024 github.com/couchbase/goxdcr/parts.(*XmemNozzle).processData_sendbatch+0x4e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1077 1 @ 0x43dbbf 0x47345b 0x68c17e 0x745185 0x74514d 0x73d0f3 0x9534ab 0x94c9c5 0x471e01 # 0x68c17d github.com/couchbase/gomemcached.(*MCResponse).ReceiveWithBuf+0xdd /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:169 # 0x745184 github.com/couchbase/gomemcached.(*MCResponse).Receive+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:156 # 0x74514c github.com/couchbase/gomemcached/client.getResponse+0x6c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/transport.go:35 # 0x73d0f2 github.com/couchbase/gomemcached/client.(*Client).Receive+0x52 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:333 # 0x9534aa github.com/couchbase/goxdcr/parts.(*XmemNozzle).readFromClient+0x18a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3498 # 0x94c9c4 github.com/couchbase/goxdcr/parts.(*XmemNozzle).receiveResponse+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2618 1 @ 0x43dbbf 0x47345b 0x758bb0 0x75e778 0x75e785 0x75edc5 0x761012 0x765bf9 0x765bcb 0x765c00 0x8496a5 0x849688 0x9bc657 0x9bc505 0x9bfaa6 0x9bed88 0xde2b74 0xdda105 0xdd9e85 0x9ba59d 0x471e01 # 0x758baf regexp/syntax.(*Inst).MatchRunePos+0x18f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/regexp/syntax/prog.go:249 # 0x75e777 regexp/syntax.(*Inst).MatchRune+0xa37 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/regexp/syntax/prog.go:194 # 0x75e784 regexp.(*Regexp).tryBacktrack+0xa44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/regexp/backtrack.go:210 # 0x75edc4 regexp.(*Regexp).backtrack+0x324 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/regexp/backtrack.go:353 # 0x761011 regexp.(*Regexp).doExecute+0x271 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/regexp/exec.go:535 # 0x765bf8 regexp.(*Regexp).doMatch+0x78 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/regexp/exec.go:514 # 0x765bca regexp.(*Regexp).MatchString+0x4a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/regexp/regexp.go:533 # 0x765bff regexp.MatchString+0x7f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/regexp/regexp.go:561 # 0x8496a4 github.com/couchbase/goxdcr/metadata.IsAReplicationId+0x44 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata/replication_spec.go:185 # 0x849687 github.com/couchbase/goxdcr/metadata.DecomposeReplicationId+0x27 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata/replication_spec.go:199 # 0x9bc656 github.com/couchbase/goxdcr/pipeline_utils.(*PerReplicationStatType).appendLabels+0xf6 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_utils/prometheusExporter.go:177 # 0x9bc504 github.com/couchbase/goxdcr/pipeline_utils.(*PerReplicationStatType).UpdateOutputBuffer+0xe4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_utils/prometheusExporter.go:169 # 0x9bfaa5 github.com/couchbase/goxdcr/pipeline_utils.(*PrometheusExporter).prepareStatsBuffer+0x225 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_utils/prometheusExporter.go:455 # 0x9bed87 github.com/couchbase/goxdcr/pipeline_utils.(*PrometheusExporter).Export+0x47 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_utils/prometheusExporter.go:373 # 0xde2b73 github.com/couchbase/goxdcr/replication_manager.(*Adminport).doGetPrometheusStatsRequest+0xf3 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/replication_manager/adminport.go:1369 # 0xdda104 github.com/couchbase/goxdcr/replication_manager.(*Adminport).handleRequest+0x1c4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/replication_manager/adminport.go:235 # 0xdd9e84 github.com/couchbase/goxdcr/replication_manager.(*Adminport).processRequest+0x64 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/replication_manager/adminport.go:158 # 0x9ba59c github.com/couchbase/goxdcr/gen_server.(*GenServer).run+0x15c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/gen_server/gen_server.go:103 1 @ 0x43dbbf 0x47345b 0x7ee7eb 0x937656 0x93723a 0x936649 0x90f4c2 0x921da2 0x471e01 # 0x7ee7ea github.com/couchbase/goxdcr/base.NewUleb128+0x20a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/simple_utils.go:1739 # 0x937655 github.com/couchbase/goxdcr/parts.(*Router).prepareMcRequest+0x2f5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/router.go:1786 # 0x937239 github.com/couchbase/goxdcr/parts.(*Router).RouteCollection+0x4d9 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/router.go:1748 # 0x936648 github.com/couchbase/goxdcr/parts.(*Router).Route+0x648 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/router.go:1679 # 0x90f4c1 github.com/couchbase/goxdcr/connector.(*Router).Forward+0x141 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/connector/router.go:113 # 0x921da1 github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0xcc1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1102 1 @ 0x43dbbf 0x47345b 0x7ffecd 0x9522bb 0x95335d 0x94c9c5 0x471e01 # 0x7ffecc github.com/couchbase/goxdcr/base.(*XmemClient).GetConn+0x16c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/xmem_client.go:151 # 0x9522ba github.com/couchbase/goxdcr/parts.(*XmemNozzle).getConn+0x9a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3298 # 0x95335c github.com/couchbase/goxdcr/parts.(*XmemNozzle).readFromClient+0x3c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3488 # 0x94c9c4 github.com/couchbase/goxdcr/parts.(*XmemNozzle).receiveResponse+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2618 1 @ 0x43dbbf 0x47345b 0x8c8848 0x7fd3d5 0xda6785 0xa1460e 0x471e01 # 0x8c8847 github.com/couchbase/goxdcr/utils.(*VbSeqnoMapPool).Get+0x127 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/dataObjPools.go:298 # 0x7fd3d4 github.com/couchbase/goxdcr/base.(*HighSeqnosMapType).GreenClone+0x354 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/types.go:2745 # 0xda6784 github.com/couchbase/goxdcr/service_impl.(*Notification).Clone+0x2c4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1936 # 0xa1460d github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).watchNotificationCh+0x1cd /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:1050 1 @ 0x43dbbf 0x47345b 0x9101b9 0x921d87 0x471e01 # 0x9101b8 github.com/couchbase/goxdcr/parts.(*AbstractPart).Connector+0x38 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/abstract_part.go:51 # 0x921d86 github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0xca6 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1102 1 @ 0x43dbbf 0x47345b 0x927697 0x92768a 0x925f3f 0x924805 0x9247ec 0x471e01 # 0x927696 sync.(*RWMutex).RLock+0x96 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/rwmutex.go:69 # 0x927689 github.com/couchbase/goxdcr/parts.(*DcpNozzle).GetStreamState+0x89 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1765 # 0x925f3e github.com/couchbase/goxdcr/parts.(*DcpNozzle).getDcpStreams+0x9e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1562 # 0x924804 github.com/couchbase/goxdcr/parts.(*DcpNozzle).nonInitDcpStreams+0x284 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1587 # 0x9247eb github.com/couchbase/goxdcr/parts.(*DcpNozzle).startUprStreams+0x26b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1355 1 @ 0x43dbbf 0x47345b 0x940151 0x941648 0x93f449 0x93f025 0x471e01 # 0x940150 github.com/couchbase/goxdcr/parts.(*XmemNozzle).batchSetMetaWithRetry+0x810 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1212 # 0x941647 github.com/couchbase/goxdcr/parts.(*XmemNozzle).sendSetMeta_internal+0x47 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1448 # 0x93f448 github.com/couchbase/goxdcr/parts.(*XmemNozzle).processBatch+0x48 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1121 # 0x93f024 github.com/couchbase/goxdcr/parts.(*XmemNozzle).processData_sendbatch+0x4e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1077 1 @ 0x43f4b6 0x40b88e 0x40b43d 0xdd9bdb 0xdd9ba3 0x471e01 # 0xdd9bda github.com/couchbase/goxdcr/gen_server.(*GenServer).SendMsg_async+0x33a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/gen_server/gen_server.go:213 # 0xdd9ba2 github.com/couchbase/goxdcr/replication_manager.(*Adminport).Start+0x302 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/replication_manager/adminport.go:131 1 @ 0x43f4b6 0x40c69d 0x40c198 0xe035f3 0x43f087 0x471e01 # 0xe035f2 main.main+0x14d2 goproj/src/github.com/couchbase/goxdcr/main/main.go:275 # 0x43f086 runtime.main+0x206 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/proc.go:250 1 @ 0x43f4b6 0x437d57 0x46c309 0x4dd352 0x4de739 0x4de727 0x5d7749 0x5e78a5 0x4bb29a 0x68c179 0x68c143 0x745185 0x74514d 0x73cb89 0x73d5e5 0x8cd5f8 0x8cce9c 0x8dc6bb 0x8dc47b 0x8ccb08 0x8cd07f 0x9e3f5e 0x9e324f 0x48d322 0x9e2eb8 0x9e2e7e 0x9e2085 0x7e7482 0x471e01 # 0x46c308 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dd351 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4de738 internal/poll.(*pollDesc).waitRead+0x298 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4de726 internal/poll.(*FD).Read+0x286 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:167 # 0x5d7748 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e78a4 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x4bb299 io.ReadAtLeast+0x99 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:332 # 0x68c178 io.ReadFull+0xd8 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:351 # 0x68c142 github.com/couchbase/gomemcached.(*MCResponse).ReceiveWithBuf+0xa2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:169 # 0x745184 github.com/couchbase/gomemcached.(*MCResponse).Receive+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:156 # 0x74514c github.com/couchbase/gomemcached/client.getResponse+0x6c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/transport.go:35 # 0x73cb88 github.com/couchbase/gomemcached/client.(*Client).Send+0x68 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:280 # 0x73d5e4 github.com/couchbase/gomemcached/client.(*Client).EnableFeatures+0x1c4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:393 # 0x8cd5f7 github.com/couchbase/goxdcr/utils.(*Utilities).SendHELOWithFeatures+0x217 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:624 # 0x8cce9b github.com/couchbase/goxdcr/utils.(*Utilities).GetRemoteMemcachedConnectionWFeatures.func1+0x27b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:544 # 0x8dc6ba github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithOriginalError+0xba /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2587 # 0x8dc47a github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutor+0x3a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2575 # 0x8ccb07 github.com/couchbase/goxdcr/utils.(*Utilities).GetRemoteMemcachedConnectionWFeatures+0x1c7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:554 # 0x8cd07e github.com/couchbase/goxdcr/utils.(*Utilities).GetRemoteMemcachedConnection+0x9e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:568 # 0x9e3f5d github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).getNewMemcachedClient+0x49d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:710 # 0x9e324e github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).initConnections.func1+0x30e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:621 # 0x48d321 sync.(*Once).doSlow+0xc1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:74 # 0x9e2eb7 sync.(*Once).Do+0x57 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:65 # 0x9e2e7d github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).initConnections+0x1d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:609 # 0x9e2084 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).initConnBg.func1+0x64 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:536 # 0x7e7481 github.com/couchbase/goxdcr/base.ExecWithTimeout.func1+0x21 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/simple_utils.go:48 1 @ 0x43f4b6 0x437d57 0x46c309 0x4dd352 0x4de739 0x4de727 0x5d7749 0x5e78a5 0x4bb29a 0x68c179 0x68c143 0x745185 0x74514d 0x73cb89 0x73f9f2 0x7e0593 0x7e352d 0x7e0095 0x7e001b 0x955822 0x8dc98a 0x955345 0x94b06f 0x94bf6c 0x93cb4b 0x9c92e2 0x9c973a 0x471e01 # 0x46c308 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dd351 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4de738 internal/poll.(*pollDesc).waitRead+0x298 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4de726 internal/poll.(*FD).Read+0x286 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:167 # 0x5d7748 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e78a4 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x4bb299 io.ReadAtLeast+0x99 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:332 # 0x68c178 io.ReadFull+0xd8 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:351 # 0x68c142 github.com/couchbase/gomemcached.(*MCResponse).ReceiveWithBuf+0xa2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:169 # 0x745184 github.com/couchbase/gomemcached.(*MCResponse).Receive+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:156 # 0x74514c github.com/couchbase/gomemcached/client.getResponse+0x6c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/transport.go:35 # 0x73cb88 github.com/couchbase/gomemcached/client.(*Client).Send+0x68 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:280 # 0x73f9f1 github.com/couchbase/gomemcached/client.(*Client).SelectBucket+0x91 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:767 # 0x7e0592 github.com/couchbase/goxdcr/base.authClient+0x212 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:251 # 0x7e352c github.com/couchbase/goxdcr/base.NewConn+0x1ec /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:612 # 0x7e0094 github.com/couchbase/goxdcr/base.(*connPool).newConn+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:208 # 0x7e001a github.com/couchbase/goxdcr/base.(*connPool).GetNew+0x1a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:204 # 0x955821 github.com/couchbase/goxdcr/parts.(*XmemNozzle).getClientWithRetry.func1+0x2a1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3774 # 0x8dc989 github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithFinishSignal+0x109 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2615 # 0x955344 github.com/couchbase/goxdcr/parts.(*XmemNozzle).getClientWithRetry+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3777 # 0x94b06e github.com/couchbase/goxdcr/parts.(*XmemNozzle).initializeConnection+0x42e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2415 # 0x94bf6b github.com/couchbase/goxdcr/parts.(*XmemNozzle).initialize+0x72b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2550 # 0x93cb4a github.com/couchbase/goxdcr/parts.(*XmemNozzle).Start+0x1ca /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:807 # 0x9c92e1 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart+0x321 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:233 # 0x9c9739 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart.func1+0xb9 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:209 1 @ 0x43f4b6 0x437d57 0x46c309 0x4dd352 0x4de739 0x4de727 0x5d7749 0x5e78a5 0x4bb29a 0x68c179 0x68c143 0x745185 0x74514d 0x73d0f3 0x747fdd 0x7488af 0x7483af 0x91db8b 0x91e6c7 0x91f78a 0x9c92e2 0x9c8d5f 0x471e01 # 0x46c308 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dd351 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4de738 internal/poll.(*pollDesc).waitRead+0x298 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4de726 internal/poll.(*FD).Read+0x286 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:167 # 0x5d7748 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e78a4 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x4bb299 io.ReadAtLeast+0x99 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:332 # 0x68c178 io.ReadFull+0xd8 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:351 # 0x68c142 github.com/couchbase/gomemcached.(*MCResponse).ReceiveWithBuf+0xa2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:169 # 0x745184 github.com/couchbase/gomemcached.(*MCResponse).Receive+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:156 # 0x74514c github.com/couchbase/gomemcached/client.getResponse+0x6c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/transport.go:35 # 0x73d0f2 github.com/couchbase/gomemcached/client.(*Client).Receive+0x52 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:333 # 0x747fdc github.com/couchbase/gomemcached/client.sendMcRequestSync+0x3c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/upr_feed.go:450 # 0x7488ae github.com/couchbase/gomemcached/client.(*UprFeed).uprOpen+0x1ee /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/upr_feed.go:529 # 0x7483ae github.com/couchbase/gomemcached/client.(*UprFeed).UprOpenWithFeatures+0x8e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/upr_feed.go:481 # 0x91db8a github.com/couchbase/goxdcr/parts.(*DcpNozzle).initializeUprFeed+0x2ca /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:575 # 0x91e6c6 github.com/couchbase/goxdcr/parts.(*DcpNozzle).initialize+0x5a6 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:739 # 0x91f789 github.com/couchbase/goxdcr/parts.(*DcpNozzle).Start+0x189 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:790 # 0x9c92e1 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart+0x321 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:233 # 0x9c8d5e github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPartsWithTimeout.func1.1+0xde /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:153 1 @ 0x43f4b6 0x437d57 0x46c309 0x4dd352 0x4de739 0x4de727 0x5d7749 0x5e78a5 0x4bb29a 0x68c179 0x68c143 0x745185 0x74514d 0x743176 0x8e05f5 0xd9e5e3 0xda2771 0xda0ca5 0x471e01 # 0x46c308 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dd351 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4de738 internal/poll.(*pollDesc).waitRead+0x298 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4de726 internal/poll.(*FD).Read+0x286 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:167 # 0x5d7748 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e78a4 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x4bb299 io.ReadAtLeast+0x99 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:332 # 0x68c178 io.ReadFull+0xd8 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:351 # 0x68c142 github.com/couchbase/gomemcached.(*MCResponse).ReceiveWithBuf+0xa2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:169 # 0x745184 github.com/couchbase/gomemcached.(*MCResponse).Receive+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:156 # 0x74514c github.com/couchbase/gomemcached/client.getResponse+0x6c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/transport.go:35 # 0x743175 github.com/couchbase/gomemcached/client.(*Client).StatsMapForSpecifiedStats+0x1b5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:1461 # 0x8e05f4 github.com/couchbase/goxdcr/utils.(*Utilities).GetHighSeqNos+0x154 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:3016 # 0xd9e5e2 github.com/couchbase/goxdcr/service_impl.(*BucketTopologyService).getHighSeqnosUpdater.func1+0x942 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:814 # 0xda2770 github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).updateOnce+0x570 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1362 # 0xda0ca4 github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).run.func2+0x1e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1176 1 @ 0x43f4b6 0x437d57 0x46c309 0x4dd352 0x4de739 0x4de727 0x5d7749 0x5e78a5 0x69373b 0x8c3765 0x6827e8 0x6823db 0x682018 0x8c2a05 0x8c2046 0x8c1b9b 0x8c12c7 0x8c3e7b 0x8c4578 0x8c68dd 0x8c6bc5 0x471e01 # 0x46c308 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dd351 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4de738 internal/poll.(*pollDesc).waitRead+0x298 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4de726 internal/poll.(*FD).Read+0x286 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:167 # 0x5d7748 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e78a4 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x69373a bufio.(*Reader).Read+0x1ba /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/bufio/bufio.go:237 # 0x8c3764 github.com/couchbase/cbauth/revrpc.(*minirwc).Read+0x24 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/revrpc/revrpc.go:102 # 0x6827e7 encoding/json.(*Decoder).refill+0x187 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/encoding/json/stream.go:165 # 0x6823da encoding/json.(*Decoder).readValue+0xba /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/encoding/json/stream.go:140 # 0x682017 encoding/json.(*Decoder).Decode+0x77 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/encoding/json/stream.go:63 # 0x8c2a04 net/rpc/jsonrpc.(*serverCodec).ReadRequestHeader+0x84 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/rpc/jsonrpc/server.go:66 # 0x8c2045 net/rpc.(*Server).readRequestHeader+0x65 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/rpc/server.go:588 # 0x8c1b9a net/rpc.(*Server).readRequest+0x3a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/rpc/server.go:548 # 0x8c12c6 net/rpc.(*Server).ServeCodec+0x86 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/rpc/server.go:463 # 0x8c3e7a github.com/couchbase/cbauth/revrpc.(*Service).Run+0x5ba /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/revrpc/revrpc.go:192 # 0x8c4577 github.com/couchbase/cbauth/revrpc.BabysitService+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/revrpc/revrpc.go:288 # 0x8c68dc github.com/couchbase/cbauth.runRPCForSvc+0xbc /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/default.go:57 # 0x8c6bc4 github.com/couchbase/cbauth.startDefault.func1+0x24 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/default.go:66 1 @ 0x43f4b6 0x437d57 0x46c309 0x4dd352 0x4de739 0x4de727 0x5d7749 0x5e78a5 0x70753f 0x471e01 # 0x46c308 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dd351 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4de738 internal/poll.(*pollDesc).waitRead+0x298 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4de726 internal/poll.(*FD).Read+0x286 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:167 # 0x5d7748 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e78a4 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x70753e net/http.(*connReader).backgroundRead+0x3e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/server.go:674 1 @ 0x43f4b6 0x437d57 0x46c309 0x4dd352 0x4e2c5d 0x4e2c4b 0x5d98d5 0x5f0765 0x5ef85d 0x7123a5 0x711fdd 0x9b8ba8 0x471e01 # 0x46c308 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dd351 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x4e2c5c internal/poll.(*pollDesc).waitRead+0x2bc /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:89 # 0x4e2c4a internal/poll.(*FD).Accept+0x2aa /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:614 # 0x5d98d4 net.(*netFD).accept+0x34 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_unix.go:172 # 0x5f0764 net.(*TCPListener).accept+0x24 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/tcpsock_posix.go:148 # 0x5ef85c net.(*TCPListener).Accept+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/tcpsock.go:297 # 0x7123a4 net/http.(*Server).Serve+0x384 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/server.go:3059 # 0x711fdc net/http.(*Server).ListenAndServe+0x7c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/server.go:2988 # 0x9b8ba7 github.com/couchbase/goxdcr/adminport.(*httpServer).Start.func1+0xe7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/adminport/admin_httpd.go:93 1 @ 0x43f4b6 0x437d57 0x46c309 0x4dd352 0x5d9179 0x5d915d 0x5d9181 0x5ecb05 0x5ec632 0x5e3b98 0x5f02cf 0x5f0169 0x5ce992 0x5ce2e5 0x5cd953 0x5cd36c 0x5ccc07 0x5ccc08 0x73c52a 0x73c5ae 0x7e33e8 0x7e0095 0x7e001b 0x955822 0x8dc98a 0x955345 0x94b06f 0x94bf6c 0x93cb4b 0x9c92e2 0x9c973a 0x471e01 # 0x46c308 internal/poll.runtime_pollWait+0x88 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/netpoll.go:306 # 0x4dd351 internal/poll.(*pollDesc).wait+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:84 # 0x5d9178 internal/poll.(*pollDesc).waitWrite+0x798 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:93 # 0x5d915c internal/poll.(*FD).WaitWrite+0x77c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:741 # 0x5d9180 net.(*netFD).connect+0x7a0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_unix.go:141 # 0x5ecb04 net.(*netFD).dial+0x3c4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/sock_posix.go:151 # 0x5ec631 net.socket+0x2d1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/sock_posix.go:70 # 0x5e3b97 net.internetSocket+0xf7 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/ipsock_posix.go:142 # 0x5f02ce net.(*sysDialer).doDialTCP+0xee /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/tcpsock_posix.go:74 # 0x5f0168 net.(*sysDialer).dialTCP+0x68 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/tcpsock_posix.go:64 # 0x5ce991 net.(*sysDialer).dialSingle+0x1f1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:580 # 0x5ce2e4 net.(*sysDialer).dialSerial+0x244 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:548 # 0x5cd952 net.(*sysDialer).dialParallel+0x412 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:449 # 0x5cd36b net.(*Dialer).DialContext+0x70b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:440 # 0x5ccc06 net.(*Dialer).Dial+0x86 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:367 # 0x5ccc07 net.DialTimeout+0x87 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/dial.go:349 # 0x73c529 github.com/couchbase/gomemcached/client.glob..func1+0x29 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:189 # 0x73c5ad github.com/couchbase/gomemcached/client.Connect+0x2d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:195 # 0x7e33e7 github.com/couchbase/goxdcr/base.NewConn+0xa7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:605 # 0x7e0094 github.com/couchbase/goxdcr/base.(*connPool).newConn+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:208 # 0x7e001a github.com/couchbase/goxdcr/base.(*connPool).GetNew+0x1a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/connection_pool.go:204 # 0x955821 github.com/couchbase/goxdcr/parts.(*XmemNozzle).getClientWithRetry.func1+0x2a1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3774 # 0x8dc989 github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithFinishSignal+0x109 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2615 # 0x955344 github.com/couchbase/goxdcr/parts.(*XmemNozzle).getClientWithRetry+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3777 # 0x94b06e github.com/couchbase/goxdcr/parts.(*XmemNozzle).initializeConnection+0x42e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2415 # 0x94bf6b github.com/couchbase/goxdcr/parts.(*XmemNozzle).initialize+0x72b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2550 # 0x93cb4a github.com/couchbase/goxdcr/parts.(*XmemNozzle).Start+0x1ca /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:807 # 0x9c92e1 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart+0x321 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:233 # 0x9c9739 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart.func1+0xb9 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:209 1 @ 0x43f4b6 0x44fa7e 0x721d46 0x71dc5e 0x705e59 0x6c6977 0x6c61fb 0x6c84fb 0x8d9413 0x8d940b 0x8d8693 0x8d2965 0xd9946b 0xda2771 0xda0ca5 0x471e01 # 0x721d45 net/http.(*Transport).getConn+0x5c5 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:1382 # 0x71dc5d net/http.(*Transport).roundTrip+0x79d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:590 # 0x705e58 net/http.(*Transport).RoundTrip+0x18 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/roundtrip.go:17 # 0x6c6976 net/http.send+0x5f6 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:252 # 0x6c61fa net/http.(*Client).send+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:176 # 0x6c84fa net/http.(*Client).do+0x8fa /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:716 # 0x8d9412 net/http.(*Client).Do+0x52 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:582 # 0x8d940a github.com/couchbase/goxdcr/utils.(*Utilities).doRestCall+0x4a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2234 # 0x8d8692 github.com/couchbase/goxdcr/utils.(*Utilities).QueryRestApiWithAuth+0x1b2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2096 # 0x8d2964 github.com/couchbase/goxdcr/utils.(*Utilities).GetBucketInfo+0x324 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:1232 # 0xd9946a github.com/couchbase/goxdcr/service_impl.(*BucketTopologyService).getLocalBucketTopologyUpdater.func1+0x38a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:354 # 0xda2770 github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).updateOnce+0x570 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1362 # 0xda0ca4 github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).run.func2+0x1e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1176 1 @ 0x43f4b6 0x44fa7e 0x729d94 0x71dcba 0x705e59 0x6c6977 0x6c61fb 0x6c84fb 0x8d9413 0x8d940b 0x8d8693 0x8d2965 0x8ded31 0x9b36f1 0x9b43fb 0xa2e517 0xa2e07e 0xdf70f5 0x471e01 # 0x729d93 net/http.(*persistConn).roundTrip+0x993 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:2638 # 0x71dcb9 net/http.(*Transport).roundTrip+0x7f9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:603 # 0x705e58 net/http.(*Transport).RoundTrip+0x18 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/roundtrip.go:17 # 0x6c6976 net/http.send+0x5f6 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:252 # 0x6c61fa net/http.(*Client).send+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:176 # 0x6c84fa net/http.(*Client).do+0x8fa /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:716 # 0x8d9412 net/http.(*Client).Do+0x52 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:582 # 0x8d940a github.com/couchbase/goxdcr/utils.(*Utilities).doRestCall+0x4a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2234 # 0x8d8692 github.com/couchbase/goxdcr/utils.(*Utilities).QueryRestApiWithAuth+0x1b2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2096 # 0x8d2964 github.com/couchbase/goxdcr/utils.(*Utilities).GetBucketInfo+0x324 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:1232 # 0x8ded30 github.com/couchbase/goxdcr/utils.(*Utilities).VerifyTargetBucket+0x170 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2844 # 0x9b36f0 github.com/couchbase/goxdcr/metadata_svc.(*ReplicationSpecService).gcTargetBucket+0xb0 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/replication_spec_service.go:1427 # 0x9b43fa github.com/couchbase/goxdcr/metadata_svc.(*ReplicationSpecService).ValidateAndGC+0x5a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/replication_spec_service.go:1504 # 0xa2e516 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineManager).validateAndGCSpecs+0x136 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:339 # 0xa2e07d github.com/couchbase/goxdcr/pipeline_manager.(*PipelineManager).CheckPipelines+0x1d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:300 # 0xdf70f4 github.com/couchbase/goxdcr/replication_manager.(*replicationManager).checkReplicationStatus+0x1d4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/replication_manager/replication_manager.go:449 1 @ 0x43f4b6 0x44fa7e 0x729d94 0x71dcba 0x705e59 0x6c6977 0x6c61fb 0x6c84fb 0x8d9413 0x8d940b 0x8d8693 0x8f2c42 0x905372 0x90534c 0x9075ff 0x471e01 # 0x729d93 net/http.(*persistConn).roundTrip+0x993 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:2638 # 0x71dcb9 net/http.(*Transport).roundTrip+0x7f9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/transport.go:603 # 0x705e58 net/http.(*Transport).RoundTrip+0x18 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/roundtrip.go:17 # 0x6c6976 net/http.send+0x5f6 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:252 # 0x6c61fa net/http.(*Client).send+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:176 # 0x6c84fa net/http.(*Client).do+0x8fa /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:716 # 0x8d9412 net/http.(*Client).Do+0x52 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/client.go:582 # 0x8d940a github.com/couchbase/goxdcr/utils.(*Utilities).doRestCall+0x4a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2234 # 0x8d8692 github.com/couchbase/goxdcr/utils.(*Utilities).QueryRestApiWithAuth+0x1b2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2096 # 0x8f2c41 github.com/couchbase/goxdcr/peerToPeer.generateRequest.func1+0x321 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerMsgs.go:379 # 0x905371 github.com/couchbase/goxdcr/peerToPeer.(*RequestCommon).CallBack+0x24f1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerMsgs.go:120 # 0x90534b github.com/couchbase/goxdcr/peerToPeer.(*VBMasterCheckHandler).handleRequest+0x24cb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/vbMasterCheckHandler.go:241 # 0x9075fe github.com/couchbase/goxdcr/peerToPeer.(*VBMasterCheckHandler).handler.func1+0x13e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/vbMasterCheckHandler.go:335 1 @ 0x43f4b6 0x44fa7e 0x7e727f 0x927b09 0x471e01 # 0x7e727e github.com/couchbase/goxdcr/base.ExecWithTimeout+0x19e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/simple_utils.go:56 # 0x927b08 github.com/couchbase/goxdcr/parts.(*DcpNozzle).checkInactiveUprStreams+0x1a8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1801 1 @ 0x43f4b6 0x44fa7e 0x87ab45 0x471e01 # 0x87ab44 github.com/couchbase/cbauth/cbauthimpl.(*cfgChangeNotifier).loop+0x84 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/cbauthimpl/impl.go:309 1 @ 0x43f4b6 0x44fa7e 0x87b0e7 0x471e01 # 0x87b0e6 github.com/couchbase/cbauth/cbauthimpl.(*tlsNotifier).loop+0x66 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/cbauth/cbauthimpl/impl.go:389 1 @ 0x43f4b6 0x44fa7e 0x8e6eaf 0x471e01 # 0x8e6eae github.com/couchbase/goxdcr/peerToPeer.(*DiscoveryHandler).handler+0xae /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/discoveryHandler.go:55 1 @ 0x43f4b6 0x44fa7e 0x8fb8b0 0x471e01 # 0x8fb8af github.com/couchbase/goxdcr/peerToPeer.(*PeriodicPushHandler).handler.func2+0x8f /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/periodicPushHandler.go:80 1 @ 0x43f4b6 0x44fa7e 0x8fba19 0x471e01 # 0x8fba18 github.com/couchbase/goxdcr/peerToPeer.(*PeriodicPushHandler).handler.func1+0x98 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/periodicPushHandler.go:62 1 @ 0x43f4b6 0x44fa7e 0x9073f0 0x471e01 # 0x9073ef github.com/couchbase/goxdcr/peerToPeer.(*VBMasterCheckHandler).handler.func2+0x8f /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/vbMasterCheckHandler.go:345 1 @ 0x43f4b6 0x44fa7e 0x9d45df 0x471e01 # 0x9d45de github.com/couchbase/goxdcr/supervisor.(*GenericSupervisor).waitForResponse+0x21e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/supervisor/generic_supervisor.go:288 1 @ 0x43f4b6 0x44fa7e 0xa14125 0xa14135 0xa116c2 0xa1158f 0xa10933 0xa1010f 0xa0f225 0xa0e745 0x471e01 # 0xa14124 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).getNotification+0xc4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:1029 # 0xa14134 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).subscribeToBucketTopologySvc.func1.2+0xd4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:975 # 0xa116c1 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).calculateChangesLeftAndDocsProcessedMainPipeline+0x101 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:670 # 0xa1158e github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).calculateChangesLeftAndDocProcessed+0x10e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:654 # 0xa10932 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).processCalculatedStats+0x72 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:548 # 0xa1010e github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).processRawStats+0x7ee /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:528 # 0xa0f224 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).updateStatsOnce+0x144 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:355 # 0xa0e744 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).updateStats+0x3c4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:303 1 @ 0x43f4b6 0x44fa7e 0xa3fda5 0x471e01 # 0xa3fda4 github.com/couchbase/goxdcr/resource_manager.(*ResourceManager).manageResources+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/resource_manager/resource_manager.go:514 1 @ 0x43f4b6 0x44fa7e 0xa40845 0x471e01 # 0xa40844 github.com/couchbase/goxdcr/resource_manager.(*ResourceManager).logStats+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/resource_manager/resource_manager.go:599 1 @ 0x43f4b6 0x44fa7e 0xa5217d 0x471e01 # 0xa5217c github.com/couchbase/goxdcr/backfill_manager.(*BackfillMgr).runRetryMonitor+0xdc /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/backfill_manager/backfill_manager.go:1651 1 @ 0x43f4b6 0x44fa7e 0xdc2585 0x471e01 # 0xdc2584 github.com/couchbase/goxdcr/service_impl.(*ThroughputThrottler).update+0x244 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/throughput_throttler_service.go:101 1 @ 0x43f4b6 0x44fa7e 0xdc2ba6 0x471e01 # 0xdc2ba5 github.com/couchbase/goxdcr/service_impl.(*ThroughputThrottler).clearReassignedTokens+0x245 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/throughput_throttler_service.go:187 1 @ 0x43f4b6 0x44fa7e 0xdc383c 0x471e01 # 0xdc383b github.com/couchbase/goxdcr/service_impl.(*ThroughputThrottler).logStats+0x1fb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/throughput_throttler_service.go:360 1 @ 0x43f4b6 0x44fa7e 0xdfc425 0x471e01 # 0xdfc424 github.com/couchbase/goxdcr/replication_manager.logMemStats+0x1c4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/replication_manager/replication_manager.go:1068 1 @ 0x43f4b6 0x450b6f 0x450b46 0x46db27 0x48ea8b 0x8efe45 0x8ff84a 0x8ff6e5 0x471e01 # 0x46db26 sync.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:62 # 0x48ea8a sync.(*WaitGroup).Wait+0x4a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/waitgroup.go:116 # 0x8efe44 github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).sendPeriodicPushRequest+0x1c4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:690 # 0x8ff849 github.com/couchbase/goxdcr/peerToPeer.(*ReplicaReplicatorImpl).pullAndSend+0xc9 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/replicaReplicator.go:271 # 0x8ff6e4 github.com/couchbase/goxdcr/peerToPeer.(*ReplicaReplicatorImpl).run+0x204 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/replicaReplicator.go:243 1 @ 0x43f4b6 0x450b6f 0x450b46 0x46db27 0x48ea8b 0x9d3912 0x471e01 # 0x46db26 sync.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:62 # 0x48ea8a sync.(*WaitGroup).Wait+0x4a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/waitgroup.go:116 # 0x9d3911 github.com/couchbase/goxdcr/supervisor.(*GenericSupervisor).supervising+0x111 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/supervisor/generic_supervisor.go:213 1 @ 0x43f4b6 0x450b6f 0x450b46 0x46db87 0x4dcdd2 0x4e06f2 0x4e0703 0x4e9465 0x4e945b 0x5118e6 0x50ef6a 0x50f0dd 0x5121fc 0x7e72ad 0x7e728a 0x90eb73 0x48d322 0x90ea5b 0x90ea1b 0x9cd2e5 0xa302f8 0xa39375 0xa379c9 0x48d322 0xa373c7 0xa37395 0x471e01 # 0x46db86 internal/poll.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:67 # 0x4dcdd1 internal/poll.(*fdMutex).rwlock+0xd1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:154 # 0x4e06f1 internal/poll.(*FD).writeLock+0x71 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:239 # 0x4e0702 internal/poll.(*FD).Write+0x82 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:370 # 0x4e9464 os.(*File).write+0x64 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file_posix.go:48 # 0x4e945a os.(*File).Write+0x5a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file.go:175 # 0x5118e5 github.com/couchbase/goxdcr/log.(*LogWriter).Write+0x25 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:94 # 0x50ef69 log.(*Logger).Output+0x349 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:194 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0x7e72ac github.com/couchbase/goxdcr/log.(*CommonLogger).Infof+0x1cc /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:193 # 0x7e7289 github.com/couchbase/goxdcr/base.ExecWithTimeout+0x1a9 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/simple_utils.go:60 # 0x90eb72 github.com/couchbase/goxdcr/component.(*AsyncComponentEventListenerImpl).Stop.func1+0xd2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/component/async_listener.go:87 # 0x48d321 sync.(*Once).doSlow+0xc1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:74 # 0x90ea5a sync.(*Once).Do+0x5a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:65 # 0x90ea1a github.com/couchbase/goxdcr/component.(*AsyncComponentEventListenerImpl).Stop+0x1a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/component/async_listener.go:85 # 0x9cd2e4 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).Stop+0x2e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:549 # 0xa302f7 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineManager).StopPipeline+0x2b7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:609 # 0xa39374 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).update+0x274 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1870 # 0xa379c8 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run.func1+0x5c8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1610 # 0x48d321 sync.(*Once).doSlow+0xc1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:74 # 0xa373c6 sync.(*Once).Do+0x46 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:65 # 0xa37394 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run+0x14 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1588 1 @ 0x43f4b6 0x450b6f 0x450b46 0x46db87 0x4dcdd2 0x4e06f2 0x4e0703 0x4e9465 0x4e945b 0x5118e6 0x50ef6a 0x50f0dd 0x5121fc 0x7e7830 0x7e77cd 0x9b90e5 0x9b9c9d 0x710689 0x711e36 0x70d812 0x471e01 # 0x46db86 internal/poll.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:67 # 0x4dcdd1 internal/poll.(*fdMutex).rwlock+0xd1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:154 # 0x4e06f1 internal/poll.(*FD).writeLock+0x71 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:239 # 0x4e0702 internal/poll.(*FD).Write+0x82 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:370 # 0x4e9464 os.(*File).write+0x64 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file_posix.go:48 # 0x4e945a os.(*File).Write+0x5a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file.go:175 # 0x5118e5 github.com/couchbase/goxdcr/log.(*LogWriter).Write+0x25 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:94 # 0x50ef69 log.(*Logger).Output+0x349 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:194 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0x7e782f github.com/couchbase/goxdcr/log.(*CommonLogger).Infof+0x36f /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:193 # 0x7e77cc github.com/couchbase/goxdcr/base.ExecWithTimeout3+0x30c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/simple_utils.go:126 # 0x9b90e4 github.com/couchbase/goxdcr/adminport.(*httpServer).systemHandler+0x1c4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/adminport/admin_httpd.go:131 # 0x9b9c9c github.com/couchbase/goxdcr/adminport.(*Handler).ServeHTTP+0x13c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/adminport/admin_httpd.go:204 # 0x710688 net/http.(*ServeMux).ServeHTTP+0x148 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/server.go:2500 # 0x711e35 net/http.serverHandler.ServeHTTP+0x315 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/server.go:2936 # 0x70d811 net/http.(*conn).serve+0x611 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/http/server.go:1995 1 @ 0x43f4b6 0x450b6f 0x450b46 0x46db87 0x4dcdd2 0x4e06f2 0x4e0703 0x4e9465 0x4e945b 0x5118e6 0x50ef6a 0x50f0dd 0x5121fc 0x8cdb0b 0x8cda74 0x8cce9c 0x8dc6bb 0x8dc47b 0x8ccb08 0x8cc3d0 0x91d65d 0x91e29a 0x91f78a 0x9c92e2 0x9c8d5f 0x471e01 # 0x46db86 internal/poll.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:67 # 0x4dcdd1 internal/poll.(*fdMutex).rwlock+0xd1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:154 # 0x4e06f1 internal/poll.(*FD).writeLock+0x71 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:239 # 0x4e0702 internal/poll.(*FD).Write+0x82 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:370 # 0x4e9464 os.(*File).write+0x64 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file_posix.go:48 # 0x4e945a os.(*File).Write+0x5a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file.go:175 # 0x5118e5 github.com/couchbase/goxdcr/log.(*LogWriter).Write+0x25 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:94 # 0x50ef69 log.(*Logger).Output+0x349 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:194 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0x8cdb0a github.com/couchbase/goxdcr/log.(*CommonLogger).Infof+0x72a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:193 # 0x8cda73 github.com/couchbase/goxdcr/utils.(*Utilities).SendHELOWithFeatures+0x693 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:661 # 0x8cce9b github.com/couchbase/goxdcr/utils.(*Utilities).GetRemoteMemcachedConnectionWFeatures.func1+0x27b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:544 # 0x8dc6ba github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithOriginalError+0xba /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2587 # 0x8dc47a github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutor+0x3a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2575 # 0x8ccb07 github.com/couchbase/goxdcr/utils.(*Utilities).GetRemoteMemcachedConnectionWFeatures+0x1c7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:554 # 0x8cc3cf github.com/couchbase/goxdcr/utils.(*Utilities).GetMemcachedConnectionWFeatures+0x40f /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:509 # 0x91d65c github.com/couchbase/goxdcr/parts.(*DcpNozzle).initializeMemcachedClient+0x1fc /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:502 # 0x91e299 github.com/couchbase/goxdcr/parts.(*DcpNozzle).initialize+0x179 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:635 # 0x91f789 github.com/couchbase/goxdcr/parts.(*DcpNozzle).Start+0x189 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:790 # 0x9c92e1 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart+0x321 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:233 # 0x9c8d5e github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPartsWithTimeout.func1.1+0xde /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:153 1 @ 0x43f4b6 0x450b6f 0x450b46 0x46db87 0x4dcdd2 0x4e06f2 0x4e0703 0x4e9465 0x4e945b 0x5118e6 0x50ef6a 0x50f0dd 0x5121fc 0x8ed5b0 0x8ed54f 0x471e01 # 0x46db86 internal/poll.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:67 # 0x4dcdd1 internal/poll.(*fdMutex).rwlock+0xd1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:154 # 0x4e06f1 internal/poll.(*FD).writeLock+0x71 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:239 # 0x4e0702 internal/poll.(*FD).Write+0x82 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:370 # 0x4e9464 os.(*File).write+0x64 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file_posix.go:48 # 0x4e945a os.(*File).Write+0x5a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file.go:175 # 0x5118e5 github.com/couchbase/goxdcr/log.(*LogWriter).Write+0x25 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:94 # 0x50ef69 log.(*Logger).Output+0x349 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:194 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0x8ed5af github.com/couchbase/goxdcr/log.(*CommonLogger).Errorf+0x38f /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:185 # 0x8ed54e github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).sendToSpecifiedPeersOnce.func1.1+0x32e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:311 1 @ 0x43f4b6 0x450b6f 0x450b46 0x46db87 0x4dcdd2 0x4e06f2 0x4e0703 0x4e9465 0x4e945b 0x5118e6 0x50ef6a 0x50f0dd 0x5121fc 0x91f7ff 0x91f794 0x9c92e2 0x9c8d5f 0x471e01 # 0x46db86 internal/poll.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:67 # 0x4dcdd1 internal/poll.(*fdMutex).rwlock+0xd1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:154 # 0x4e06f1 internal/poll.(*FD).writeLock+0x71 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:239 # 0x4e0702 internal/poll.(*FD).Write+0x82 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:370 # 0x4e9464 os.(*File).write+0x64 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file_posix.go:48 # 0x4e945a os.(*File).Write+0x5a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file.go:175 # 0x5118e5 github.com/couchbase/goxdcr/log.(*LogWriter).Write+0x25 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:94 # 0x50ef69 log.(*Logger).Output+0x349 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:194 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0x91f7fe github.com/couchbase/goxdcr/log.(*CommonLogger).Infof+0x1fe /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:193 # 0x91f793 github.com/couchbase/goxdcr/parts.(*DcpNozzle).Start+0x193 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:794 # 0x9c92e1 github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPart+0x321 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:233 # 0x9c8d5e github.com/couchbase/goxdcr/pipeline.(*GenericPipeline).startPartsWithTimeout.func1.1+0xde /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline/generic_pipeline.go:153 1 @ 0x43f4b6 0x450b6f 0x450b46 0x46db87 0x4dcdd2 0x4e06f2 0x4e0703 0x4e9465 0x4e945b 0x5118e6 0x50ef6a 0x50f0dd 0x5121fc 0x9280a8 0x928012 0x7e7482 0x471e01 # 0x46db86 internal/poll.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:67 # 0x4dcdd1 internal/poll.(*fdMutex).rwlock+0xd1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:154 # 0x4e06f1 internal/poll.(*FD).writeLock+0x71 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:239 # 0x4e0702 internal/poll.(*FD).Write+0x82 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:370 # 0x4e9464 os.(*File).write+0x64 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file_posix.go:48 # 0x4e945a os.(*File).Write+0x5a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file.go:175 # 0x5118e5 github.com/couchbase/goxdcr/log.(*LogWriter).Write+0x25 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:94 # 0x50ef69 log.(*Logger).Output+0x349 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:194 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0x9280a7 github.com/couchbase/goxdcr/log.(*CommonLogger).Infof+0x107 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:193 # 0x928011 github.com/couchbase/goxdcr/parts.(*DcpNozzle).checkInactiveUprStreams_once+0x71 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1834 # 0x7e7481 github.com/couchbase/goxdcr/base.ExecWithTimeout.func1+0x21 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/simple_utils.go:48 1 @ 0x43f4b6 0x450b6f 0x450b46 0x46db87 0x4dcdd2 0x4e06f2 0x4e0703 0x4e9465 0x4e945b 0x5118e6 0x50ef6a 0x50f0dd 0x5121fc 0x9519af 0x9518e6 0xa0f4f6 0xa0ee3b 0x471e01 # 0x46db86 internal/poll.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:67 # 0x4dcdd1 internal/poll.(*fdMutex).rwlock+0xd1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:154 # 0x4e06f1 internal/poll.(*FD).writeLock+0x71 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:239 # 0x4e0702 internal/poll.(*FD).Write+0x82 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:370 # 0x4e9464 os.(*File).write+0x64 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file_posix.go:48 # 0x4e945a os.(*File).Write+0x5a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file.go:175 # 0x5118e5 github.com/couchbase/goxdcr/log.(*LogWriter).Write+0x25 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:94 # 0x50ef69 log.(*Logger).Output+0x349 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:194 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0x9519ae github.com/couchbase/goxdcr/log.(*CommonLogger).Infof+0x6ce /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:193 # 0x9518e5 github.com/couchbase/goxdcr/parts.(*XmemNozzle).PrintStatusSummary+0x605 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3226 # 0xa0f4f5 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).logStatsOnce+0x175 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:375 # 0xa0ee3a github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).logStats+0x23a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:336 1 @ 0x43f4b6 0x450b6f 0x450b46 0x46db87 0x4dcdd2 0x4e06f2 0x4e0703 0x4e9465 0x4e945b 0x5118e6 0x50ef6a 0x50f0dd 0x5121fc 0x98a52f 0x98a467 0x98d097 0x991a9f 0x471e01 # 0x46db86 internal/poll.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:67 # 0x4dcdd1 internal/poll.(*fdMutex).rwlock+0xd1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:154 # 0x4e06f1 internal/poll.(*FD).writeLock+0x71 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:239 # 0x4e0702 internal/poll.(*FD).Write+0x82 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:370 # 0x4e9464 os.(*File).write+0x64 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file_posix.go:48 # 0x4e945a os.(*File).Write+0x5a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file.go:175 # 0x5118e5 github.com/couchbase/goxdcr/log.(*LogWriter).Write+0x25 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:94 # 0x50ef69 log.(*Logger).Output+0x349 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:194 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0x98a52e github.com/couchbase/goxdcr/log.(*CommonLogger).Infof+0x14e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:193 # 0x98a466 github.com/couchbase/goxdcr/metadata_svc.(*refreshContext).checkAndUpdateActiveHost+0x86 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/remote_cluster_service.go:660 # 0x98d096 github.com/couchbase/goxdcr/metadata_svc.(*RemoteClusterAgent).Refresh+0x4b6 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/remote_cluster_service.go:932 # 0x991a9e github.com/couchbase/goxdcr/metadata_svc.(*RemoteClusterAgent).runPeriodicRefresh+0x19e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/remote_cluster_service.go:1619 1 @ 0x43f4b6 0x450b6f 0x450b46 0x46db87 0x4dcdd2 0x4e06f2 0x4e0703 0x4e9465 0x4e945b 0x5118e6 0x50ef6a 0x50f0dd 0x5121fc 0x9ea897 0x9ea70b 0x471e01 # 0x46db86 internal/poll.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:67 # 0x4dcdd1 internal/poll.(*fdMutex).rwlock+0xd1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:154 # 0x4e06f1 internal/poll.(*FD).writeLock+0x71 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:239 # 0x4e0702 internal/poll.(*FD).Write+0x82 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:370 # 0x4e9464 os.(*File).write+0x64 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file_posix.go:48 # 0x4e945a os.(*File).Write+0x5a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file.go:175 # 0x5118e5 github.com/couchbase/goxdcr/log.(*LogWriter).Write+0x25 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:94 # 0x50ef69 log.(*Logger).Output+0x349 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:194 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0x9ea896 github.com/couchbase/goxdcr/log.(*CommonLogger).Infof+0x216 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:193 # 0x9ea70a github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).startSeqnoGetter+0x8a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1316 1 @ 0x43f4b6 0x450b6f 0x450b46 0x46db87 0x4dcdd2 0x4e06f2 0x4e0703 0x4e9465 0x4e945b 0x5118e6 0x50ef6a 0x50f0dd 0x5121fc 0xa0ecc5 0xa0ec4f 0x471e01 # 0x46db86 internal/poll.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:67 # 0x4dcdd1 internal/poll.(*fdMutex).rwlock+0xd1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:154 # 0x4e06f1 internal/poll.(*FD).writeLock+0x71 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:239 # 0x4e0702 internal/poll.(*FD).Write+0x82 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:370 # 0x4e9464 os.(*File).write+0x64 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file_posix.go:48 # 0x4e945a os.(*File).Write+0x5a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file.go:175 # 0x5118e5 github.com/couchbase/goxdcr/log.(*LogWriter).Write+0x25 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:94 # 0x50ef69 log.(*Logger).Output+0x349 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:194 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0xa0ecc4 github.com/couchbase/goxdcr/log.(*CommonLogger).Infof+0xc4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:193 # 0xa0ec4e github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).logStats+0x4e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:323 1 @ 0x43f4b6 0x450b6f 0x450b46 0x46db87 0x4dcdd2 0x4e06f2 0x4e0703 0x4e9465 0x4e945b 0x5118e6 0x50ef6a 0x50f0dd 0x5121fc 0xa22705 0xa2267c 0xa25b67 0x471e01 # 0x46db86 internal/poll.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:67 # 0x4dcdd1 internal/poll.(*fdMutex).rwlock+0xd1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:154 # 0x4e06f1 internal/poll.(*FD).writeLock+0x71 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:239 # 0x4e0702 internal/poll.(*FD).Write+0x82 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:370 # 0x4e9464 os.(*File).write+0x64 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file_posix.go:48 # 0x4e945a os.(*File).Write+0x5a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file.go:175 # 0x5118e5 github.com/couchbase/goxdcr/log.(*LogWriter).Write+0x25 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:94 # 0x50ef69 log.(*Logger).Output+0x349 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:194 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0xa22704 github.com/couchbase/goxdcr/log.(*CommonLogger).Infof+0x144 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:193 # 0xa2267b github.com/couchbase/goxdcr/pipeline_svc.(*TopologyChangeDetectorSvc).pipelineHasStopped+0xbb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/topology_change_detector.go:223 # 0xa25b66 github.com/couchbase/goxdcr/pipeline_svc.(*TopologyChangeDetectorSvc).monitorTarget.func1+0x106 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/topology_change_detector.go:581 1 @ 0x43f4b6 0x450b6f 0x450b46 0x46db87 0x4dcdd2 0x4e06f2 0x4e0703 0x4e9465 0x4e945b 0x5118e6 0x50ef6a 0x50f0dd 0x5121fc 0xa3b985 0xa3b95e 0x471e01 # 0x46db86 internal/poll.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:67 # 0x4dcdd1 internal/poll.(*fdMutex).rwlock+0xd1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:154 # 0x4e06f1 internal/poll.(*FD).writeLock+0x71 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:239 # 0x4e0702 internal/poll.(*FD).Write+0x82 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:370 # 0x4e9464 os.(*File).write+0x64 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file_posix.go:48 # 0x4e945a os.(*File).Write+0x5a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file.go:175 # 0x5118e5 github.com/couchbase/goxdcr/log.(*LogWriter).Write+0x25 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:94 # 0x50ef69 log.(*Logger).Output+0x349 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:194 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0xa3b984 github.com/couchbase/goxdcr/log.(*CommonLogger).Infof+0x44 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:193 # 0xa3b95d github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).scheduleFutureRefresh.func1+0x1d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:2215 1 @ 0x43f4b6 0x450b6f 0x450b46 0x46db87 0x4dcdd2 0x4e06f2 0x4e0703 0x4e9465 0x4e945b 0x5118e6 0x50ef6a 0x50f23a 0x512329 0x92415f 0x9240e6 0xa0f594 0xa0ee3b 0x471e01 # 0x46db86 internal/poll.runtime_Semacquire+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:67 # 0x4dcdd1 internal/poll.(*fdMutex).rwlock+0xd1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:154 # 0x4e06f1 internal/poll.(*FD).writeLock+0x71 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_mutex.go:239 # 0x4e0702 internal/poll.(*FD).Write+0x82 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:370 # 0x4e9464 os.(*File).write+0x64 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file_posix.go:48 # 0x4e945a os.(*File).Write+0x5a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file.go:175 # 0x5118e5 github.com/couchbase/goxdcr/log.(*LogWriter).Write+0x25 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:94 # 0x50ef69 log.(*Logger).Output+0x349 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:194 # 0x50f239 log.(*Logger).Println+0x59 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:222 # 0x512328 github.com/couchbase/goxdcr/log.(*CommonLogger).logMsg+0xc8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:176 # 0x92415e github.com/couchbase/goxdcr/log.(*CommonLogger).Info+0x1be /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:217 # 0x9240e5 github.com/couchbase/goxdcr/parts.(*DcpNozzle).PrintStatusSummary+0x145 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1311 # 0xa0f593 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).logStatsOnce+0x213 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:382 # 0xa0ee3a github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).logStats+0x23a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:336 1 @ 0x43f4b6 0x450b6f 0x450b46 0x46dc46 0x48cf85 0x48e5d6 0x48e5b5 0x7ffaa5 0x95329a 0x952689 0x94a32c 0x950c4b 0x939f54 0x9506c9 0x471e01 # 0x46dc45 sync.runtime_SemacquireMutex+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:77 # 0x48cf84 sync.(*Mutex).lockSlow+0x164 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:171 # 0x48e5d5 sync.(*Mutex).Lock+0x35 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:90 # 0x48e5b4 sync.(*RWMutex).Lock+0x14 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/rwmutex.go:147 # 0x7ffaa4 github.com/couchbase/goxdcr/base.(*XmemClient).ReportOpSuccess+0x44 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/xmem_client.go:101 # 0x953299 github.com/couchbase/goxdcr/parts.(*XmemNozzle).writeToClientWithoutThrottling+0x299 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3466 # 0x952688 github.com/couchbase/goxdcr/parts.(*XmemNozzle).writeToClient+0x388 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3340 # 0x94a32b github.com/couchbase/goxdcr/parts.(*XmemNozzle).sendSingleSetMeta+0xab /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2334 # 0x950c4a github.com/couchbase/goxdcr/parts.(*XmemNozzle).resendIfTimeout+0x46a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3051 # 0x939f53 github.com/couchbase/goxdcr/parts.(*requestBuffer).modSlot+0x53 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:296 # 0x9506c8 github.com/couchbase/goxdcr/parts.(*XmemNozzle).checkAndRepairBufferMonitor+0x348 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2995 1 @ 0x43f4b6 0x450b6f 0x450b46 0x46dc46 0x48cf85 0x48e5d6 0x48e5b5 0x950837 0x939f54 0x9506c9 0x471e01 # 0x46dc45 sync.runtime_SemacquireMutex+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:77 # 0x48cf84 sync.(*Mutex).lockSlow+0x164 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:171 # 0x48e5d5 sync.(*Mutex).Lock+0x35 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:90 # 0x48e5b4 sync.(*RWMutex).Lock+0x14 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/rwmutex.go:147 # 0x950836 github.com/couchbase/goxdcr/parts.(*XmemNozzle).resendIfTimeout+0x56 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3009 # 0x939f53 github.com/couchbase/goxdcr/parts.(*requestBuffer).modSlot+0x53 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:296 # 0x9506c8 github.com/couchbase/goxdcr/parts.(*XmemNozzle).checkAndRepairBufferMonitor+0x348 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2995 1 @ 0x43f4b6 0x450b6f 0x450b46 0x46dc46 0x48cf85 0x50ecbc 0x50ec9b 0x50f0dd 0x5121fc 0x512597 0xa22ce4 0xa25266 0x471e01 # 0x46dc45 sync.runtime_SemacquireMutex+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:77 # 0x48cf84 sync.(*Mutex).lockSlow+0x164 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:171 # 0x50ecbb sync.(*Mutex).Lock+0x9b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:90 # 0x50ec9a log.(*Logger).Output+0x7a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:175 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0x512596 github.com/couchbase/goxdcr/log.(*CommonLogger).Infof+0x36 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:193 # 0xa22ce3 github.com/couchbase/goxdcr/pipeline_svc.(*TopologyChangeDetectorSvc).handleSourceTopologyChange+0x5a3 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/topology_change_detector.go:302 # 0xa25265 github.com/couchbase/goxdcr/pipeline_svc.(*TopologyChangeDetectorSvc).monitorSource.func1+0x3a5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/topology_change_detector.go:526 1 @ 0x43f4b6 0x450b6f 0x450b46 0x46dc46 0x48cf85 0x50ecbc 0x50ec9b 0x50f0dd 0x5121fc 0x8cdb0b 0x8cda74 0x8cce9c 0x8dc6bb 0x8dc47b 0x8ccb08 0x8cd07f 0x9e3f5e 0x9e324f 0x48d322 0x9e2eb8 0x9e2e7e 0x9e2085 0x7e7482 0x471e01 # 0x46dc45 sync.runtime_SemacquireMutex+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:77 # 0x48cf84 sync.(*Mutex).lockSlow+0x164 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:171 # 0x50ecbb sync.(*Mutex).Lock+0x9b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:90 # 0x50ec9a log.(*Logger).Output+0x7a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:175 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0x8cdb0a github.com/couchbase/goxdcr/log.(*CommonLogger).Infof+0x72a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:193 # 0x8cda73 github.com/couchbase/goxdcr/utils.(*Utilities).SendHELOWithFeatures+0x693 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:661 # 0x8cce9b github.com/couchbase/goxdcr/utils.(*Utilities).GetRemoteMemcachedConnectionWFeatures.func1+0x27b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:544 # 0x8dc6ba github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithOriginalError+0xba /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2587 # 0x8dc47a github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutor+0x3a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2575 # 0x8ccb07 github.com/couchbase/goxdcr/utils.(*Utilities).GetRemoteMemcachedConnectionWFeatures+0x1c7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:554 # 0x8cd07e github.com/couchbase/goxdcr/utils.(*Utilities).GetRemoteMemcachedConnection+0x9e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:568 # 0x9e3f5d github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).getNewMemcachedClient+0x49d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:710 # 0x9e324e github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).initConnections.func1+0x30e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:621 # 0x48d321 sync.(*Once).doSlow+0xc1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:74 # 0x9e2eb7 sync.(*Once).Do+0x57 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:65 # 0x9e2e7d github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).initConnections+0x1d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:609 # 0x9e2084 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).initConnBg.func1+0x64 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:536 # 0x7e7481 github.com/couchbase/goxdcr/base.ExecWithTimeout.func1+0x21 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/simple_utils.go:48 1 @ 0x43f4b6 0x450b6f 0x450b46 0x46dc46 0x48cf85 0x50ecbc 0x50ec9b 0x50f0dd 0x5121fc 0x8dffef 0x8dff6f 0x8cc879 0x8ccd25 0x8dc6bb 0x8dc47b 0x8ccb08 0x8cd07f 0x9e3f5e 0x9e324f 0x48d322 0x9e2eb8 0x9e2e7e 0x9e2085 0x7e7482 0x471e01 # 0x46dc45 sync.runtime_SemacquireMutex+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:77 # 0x48cf84 sync.(*Mutex).lockSlow+0x164 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:171 # 0x50ecbb sync.(*Mutex).Lock+0x9b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:90 # 0x50ec9a log.(*Logger).Output+0x7a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:175 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0x8dffee github.com/couchbase/goxdcr/log.(*CommonLogger).Warnf+0xee /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:189 # 0x8dff6e github.com/couchbase/goxdcr/utils.(*Utilities).StartDiagStopwatch.func1+0x6e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2960 # 0x8cc878 github.com/couchbase/goxdcr/utils.(*Utilities).GetMemcachedRawConn+0x278 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:528 # 0x8ccd24 github.com/couchbase/goxdcr/utils.(*Utilities).GetRemoteMemcachedConnectionWFeatures.func1+0x104 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:538 # 0x8dc6ba github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithOriginalError+0xba /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2587 # 0x8dc47a github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutor+0x3a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2575 # 0x8ccb07 github.com/couchbase/goxdcr/utils.(*Utilities).GetRemoteMemcachedConnectionWFeatures+0x1c7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:554 # 0x8cd07e github.com/couchbase/goxdcr/utils.(*Utilities).GetRemoteMemcachedConnection+0x9e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:568 # 0x9e3f5d github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).getNewMemcachedClient+0x49d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:710 # 0x9e324e github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).initConnections.func1+0x30e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:621 # 0x48d321 sync.(*Once).doSlow+0xc1 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:74 # 0x9e2eb7 sync.(*Once).Do+0x57 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/once.go:65 # 0x9e2e7d github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).initConnections+0x1d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:609 # 0x9e2084 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).initConnBg.func1+0x64 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:536 # 0x7e7481 github.com/couchbase/goxdcr/base.ExecWithTimeout.func1+0x21 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/simple_utils.go:48 1 @ 0x43f4b6 0x450b6f 0x450b46 0x46dc46 0x48cf85 0x50ecbc 0x50ec9b 0x50f0dd 0x5121fc 0x8dffef 0x8dff6f 0x984044 0x963352 0x96333a 0x9608de 0x970259 0x970a32 0x9e7ea2 0xdd18e5 0x471e01 # 0x46dc45 sync.runtime_SemacquireMutex+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:77 # 0x48cf84 sync.(*Mutex).lockSlow+0x164 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:171 # 0x50ecbb sync.(*Mutex).Lock+0x9b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:90 # 0x50ec9a log.(*Logger).Output+0x7a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:175 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0x8dffee github.com/couchbase/goxdcr/log.(*CommonLogger).Warnf+0xee /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:189 # 0x8dff6e github.com/couchbase/goxdcr/utils.(*Utilities).StartDiagStopwatch.func1+0x6e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2960 # 0x984043 github.com/couchbase/goxdcr/metadata_svc.(*MetaKVMetadataSvc).Get+0x3c3 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/metakv_metadata_service.go:72 # 0x963351 github.com/couchbase/goxdcr/metadata_svc.(*MapShaRefCounter).getCollectionNsMappingsDocData+0x51 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/ShaRefCounterService.go:363 # 0x963339 github.com/couchbase/goxdcr/metadata_svc.(*MapShaRefCounter).GetMappingsDoc+0x39 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/ShaRefCounterService.go:474 # 0x9608dd github.com/couchbase/goxdcr/metadata_svc.(*ShaRefCounterService).GetMappingsDoc+0xbd /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/ShaRefCounterService.go:96 # 0x970258 github.com/couchbase/goxdcr/metadata_svc.(*CheckpointsService).loadBrokenMappingsInternal+0x58 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/checkpoints_service.go:466 # 0x970a31 github.com/couchbase/goxdcr/metadata_svc.(*CheckpointsService).CheckpointsDocs+0x311 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/checkpoints_service.go:561 # 0x9e7ea1 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).SetVBTimestamps+0x4e1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1075 # 0xdd18e4 github.com/couchbase/goxdcr/factory.(*XDCRFactory).SetStartSeqno+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/factory/xdcr_factory.go:888 1 @ 0x43f4b6 0x450b6f 0x450b46 0x46dc46 0x48cf85 0x50ecbc 0x50ec9b 0x50f0dd 0x5121fc 0x8dffef 0x8dff6f 0x984044 0x96d110 0x96eed6 0x9f41ce 0x9f182f 0x9f2a38 0x9f0f59 0x9effe6 0x471e01 # 0x46dc45 sync.runtime_SemacquireMutex+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:77 # 0x48cf84 sync.(*Mutex).lockSlow+0x164 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:171 # 0x50ecbb sync.(*Mutex).Lock+0x9b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:90 # 0x50ec9a log.(*Logger).Output+0x7a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:175 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0x8dffee github.com/couchbase/goxdcr/log.(*CommonLogger).Warnf+0xee /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:189 # 0x8dff6e github.com/couchbase/goxdcr/utils.(*Utilities).StartDiagStopwatch.func1+0x6e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2960 # 0x984043 github.com/couchbase/goxdcr/metadata_svc.(*MetaKVMetadataSvc).Get+0x3c3 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/metakv_metadata_service.go:72 # 0x96d10f github.com/couchbase/goxdcr/metadata_svc.(*CheckpointsService).checkpointsDocInternal+0x20f /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/checkpoints_service.go:120 # 0x96eed5 github.com/couchbase/goxdcr/metadata_svc.(*CheckpointsService).UpsertCheckpoints+0x315 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/checkpoints_service.go:343 # 0x9f41cd github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).persistCkptRecord+0x20d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:2150 # 0x9f182e github.com/couchbase/goxdcr/pipeline_svc.(*checkpointRecordWithLock).updateAndPersist+0x12e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1944 # 0x9f2a37 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).doCheckpoint+0xd37 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:2067 # 0x9f0f58 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).performCkpt_internal+0x738 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1894 # 0x9effe5 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).performCkpt+0x7c5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1828 1 @ 0x43f4b6 0x450b6f 0x450b46 0x46dc46 0x48cf85 0x50ecbc 0x50ec9b 0x50f0dd 0x5121fc 0x8dffef 0x8dff6f 0x986f6d 0x9709e8 0x9e7ea2 0xdd18e5 0x471e01 # 0x46dc45 sync.runtime_SemacquireMutex+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:77 # 0x48cf84 sync.(*Mutex).lockSlow+0x164 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:171 # 0x50ecbb sync.(*Mutex).Lock+0x9b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:90 # 0x50ec9a log.(*Logger).Output+0x7a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:175 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0x8dffee github.com/couchbase/goxdcr/log.(*CommonLogger).Warnf+0xee /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:189 # 0x8dff6e github.com/couchbase/goxdcr/utils.(*Utilities).StartDiagStopwatch.func1+0x6e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2960 # 0x986f6c github.com/couchbase/goxdcr/metadata_svc.(*MetaKVMetadataSvc).GetAllMetadataFromCatalog+0x2ec /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/metakv_metadata_service.go:327 # 0x9709e7 github.com/couchbase/goxdcr/metadata_svc.(*CheckpointsService).CheckpointsDocs+0x2c7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/checkpoints_service.go:550 # 0x9e7ea1 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).SetVBTimestamps+0x4e1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1075 # 0xdd18e4 github.com/couchbase/goxdcr/factory.(*XDCRFactory).SetStartSeqno+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/factory/xdcr_factory.go:888 1 @ 0x43f4b6 0x450b6f 0x450b46 0x46dc46 0x48cf85 0x50ecbc 0x50ec9b 0x50f0dd 0x5121fc 0x9edbd4 0x9edaa9 0x471e01 # 0x46dc45 sync.runtime_SemacquireMutex+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:77 # 0x48cf84 sync.(*Mutex).lockSlow+0x164 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:171 # 0x50ecbb sync.(*Mutex).Lock+0x9b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:90 # 0x50ec9a log.(*Logger).Output+0x7a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:175 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0x9edbd3 github.com/couchbase/goxdcr/log.(*CommonLogger).Infof+0x173 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:193 # 0x9edaa8 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).checkpointing+0x48 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1645 1 @ 0x43f4b6 0x450b6f 0x450b46 0x46dc46 0x48cf85 0x50ecbc 0x50ec9b 0x50f0dd 0x5121fc 0xa11c73 0xa11b9e 0xa117ea 0xa1158f 0xa10933 0xa1010f 0xa0f225 0xa0e765 0x471e01 # 0x46dc45 sync.runtime_SemacquireMutex+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:77 # 0x48cf84 sync.(*Mutex).lockSlow+0x164 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:171 # 0x50ecbb sync.(*Mutex).Lock+0x9b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/mutex.go:90 # 0x50ec9a log.(*Logger).Output+0x7a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:175 # 0x50f0dc log.(*Logger).Printf+0x5c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:204 # 0x5121fb github.com/couchbase/goxdcr/log.(*CommonLogger).logMsgf+0xfb /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:170 # 0xa11c72 github.com/couchbase/goxdcr/log.(*CommonLogger).Infof+0x112 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/log/logger.go:193 # 0xa11b9d github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).logChangesLeft+0x3d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:699 # 0xa117e9 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).calculateChangesLeftAndDocsProcessedMainPipeline+0x229 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:684 # 0xa1158e github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).calculateChangesLeftAndDocProcessed+0x10e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:654 # 0xa10932 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).processCalculatedStats+0x72 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:548 # 0xa1010e github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).processRawStats+0x7ee /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:528 # 0xa0f224 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).updateStatsOnce+0x144 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:355 # 0xa0e764 github.com/couchbase/goxdcr/pipeline_svc.(*StatisticsManager).updateStats+0x3e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/statistics_manager.go:309 1 @ 0x43f4b6 0x450b6f 0x450b46 0x46dca6 0x7ffdd4 0x7ffdb5 0x9522bb 0x95335d 0x94c9c5 0x471e01 # 0x46dca5 sync.runtime_SemacquireRWMutexR+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:82 # 0x7ffdd3 sync.(*RWMutex).RLock+0x73 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/rwmutex.go:71 # 0x7ffdb4 github.com/couchbase/goxdcr/base.(*XmemClient).GetConn+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/xmem_client.go:132 # 0x9522ba github.com/couchbase/goxdcr/parts.(*XmemNozzle).getConn+0x9a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3298 # 0x95335c github.com/couchbase/goxdcr/parts.(*XmemNozzle).readFromClient+0x3c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3488 # 0x94c9c4 github.com/couchbase/goxdcr/parts.(*XmemNozzle).receiveResponse+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2618 1 @ 0x43f4b6 0x450b6f 0x450b46 0x46dca6 0x970ab4 0x970a94 0x901b71 0x901765 0x90138b 0x471e01 # 0x46dca5 sync.runtime_SemacquireRWMutexR+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:82 # 0x970ab3 sync.(*RWMutex).RLock+0x393 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/rwmutex.go:71 # 0x970a93 github.com/couchbase/goxdcr/metadata_svc.(*CheckpointsService).CheckpointsDocs+0x373 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/metadata_svc/checkpoints_service.go:575 # 0x901b70 github.com/couchbase/goxdcr/peerToPeer.(*ReplicatorAgentImpl).FetchLatestReplicationsInfo+0x170 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/replicaReplicator.go:586 # 0x901764 github.com/couchbase/goxdcr/peerToPeer.(*ReplicatorAgentImpl).fetchAndCacheLatestReplInfo+0x24 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/replicaReplicator.go:554 # 0x90138a github.com/couchbase/goxdcr/peerToPeer.(*ReplicatorAgentImpl).run+0x6aa /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/replicaReplicator.go:533 1 @ 0x43f4b6 0x450b6f 0x450b46 0x46dd06 0x48e611 0x7ffaa5 0x95329a 0x952689 0x941525 0x9401d3 0x941648 0x93f449 0x93f025 0x471e01 # 0x46dd05 sync.runtime_SemacquireRWMutex+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:87 # 0x48e610 sync.(*RWMutex).Lock+0x70 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/rwmutex.go:152 # 0x7ffaa4 github.com/couchbase/goxdcr/base.(*XmemClient).ReportOpSuccess+0x44 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/xmem_client.go:101 # 0x953299 github.com/couchbase/goxdcr/parts.(*XmemNozzle).writeToClientWithoutThrottling+0x299 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3466 # 0x952688 github.com/couchbase/goxdcr/parts.(*XmemNozzle).writeToClient+0x388 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3340 # 0x941524 github.com/couchbase/goxdcr/parts.(*XmemNozzle).sendWithRetry+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1434 # 0x9401d2 github.com/couchbase/goxdcr/parts.(*XmemNozzle).batchSetMetaWithRetry+0x892 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1218 # 0x941647 github.com/couchbase/goxdcr/parts.(*XmemNozzle).sendSetMeta_internal+0x47 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1448 # 0x93f448 github.com/couchbase/goxdcr/parts.(*XmemNozzle).processBatch+0x48 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1121 # 0x93f024 github.com/couchbase/goxdcr/parts.(*XmemNozzle).processData_sendbatch+0x4e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1077 1 @ 0x43f4b6 0x450b6f 0x450b46 0x46dd06 0x48e611 0x7ffaa5 0x953745 0x94c9c5 0x471e01 # 0x46dd05 sync.runtime_SemacquireRWMutex+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:87 # 0x48e610 sync.(*RWMutex).Lock+0x70 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/rwmutex.go:152 # 0x7ffaa4 github.com/couchbase/goxdcr/base.(*XmemClient).ReportOpSuccess+0x44 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/xmem_client.go:101 # 0x953744 github.com/couchbase/goxdcr/parts.(*XmemNozzle).readFromClient+0x424 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3531 # 0x94c9c4 github.com/couchbase/goxdcr/parts.(*XmemNozzle).receiveResponse+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2618 1 @ 0x43f4b6 0x450b6f 0x450b46 0x46dd06 0x48e611 0x950837 0x939f54 0x9506c9 0x471e01 # 0x46dd05 sync.runtime_SemacquireRWMutex+0x25 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/sema.go:87 # 0x48e610 sync.(*RWMutex).Lock+0x70 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/rwmutex.go:152 # 0x950836 github.com/couchbase/goxdcr/parts.(*XmemNozzle).resendIfTimeout+0x56 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3009 # 0x939f53 github.com/couchbase/goxdcr/parts.(*requestBuffer).modSlot+0x53 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:296 # 0x9506c8 github.com/couchbase/goxdcr/parts.(*XmemNozzle).checkAndRepairBufferMonitor+0x348 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2995 1 @ 0x43f4b6 0x46ebf5 0xd842a5 0x471e01 # 0x46ebf4 time.Sleep+0x134 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/runtime/time.go:195 # 0xd842a4 github.com/couchbase/eventing-ee/evaluator/store.(*storage).periodicScan+0x44 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing-ee/evaluator/store/store.go:328 1 @ 0x4507e5 0x94c945 0x471e01 # 0x94c944 github.com/couchbase/goxdcr/parts.(*XmemNozzle).receiveResponse+0xe4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2608 1 @ 0x4507e5 0x95484b 0x93e5e6 0x93dbb6 0x90f615 0x921da2 0x471e01 # 0x95484a github.com/couchbase/goxdcr/parts.(*XmemNozzle).writeToDataChan+0xea /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3664 # 0x93e5e5 github.com/couchbase/goxdcr/parts.(*XmemNozzle).accumuBatch+0x3e5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:955 # 0x93dbb5 github.com/couchbase/goxdcr/parts.(*XmemNozzle).Receive+0x115 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:907 # 0x90f614 github.com/couchbase/goxdcr/connector.(*Router).Forward+0x294 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/connector/router.go:118 # 0x921da1 github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0xcc1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1102 1 @ 0x454dd7 0x9523c6 0x95237d 0x94a32c 0x950c4b 0x939f54 0x9506c9 0x471e01 # 0x9523c5 github.com/couchbase/goxdcr/base.(*Stack).Push+0xc5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/types.go:891 # 0x95237c github.com/couchbase/goxdcr/parts.(*XmemNozzle).writeToClient+0x7c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3316 # 0x94a32b github.com/couchbase/goxdcr/parts.(*XmemNozzle).sendSingleSetMeta+0xab /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2334 # 0x950c4a github.com/couchbase/goxdcr/parts.(*XmemNozzle).resendIfTimeout+0x46a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3051 # 0x939f53 github.com/couchbase/goxdcr/parts.(*requestBuffer).modSlot+0x53 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:296 # 0x9506c8 github.com/couchbase/goxdcr/parts.(*XmemNozzle).checkAndRepairBufferMonitor+0x348 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2995 1 @ 0x48e7cd 0x9359b9 0x9363e5 0x90f4c2 0x921da2 0x471e01 # 0x48e7cc sync.(*RWMutex).Unlock+0xac /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/rwmutex.go:198 # 0x9359b8 github.com/couchbase/goxdcr/parts.(*Router).ComposeMCRequest+0x738 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/router.go:1555 # 0x9363e4 github.com/couchbase/goxdcr/parts.(*Router).Route+0x3e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/router.go:1663 # 0x90f4c1 github.com/couchbase/goxdcr/connector.(*Router).Forward+0x141 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/connector/router.go:113 # 0x921da1 github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0xcc1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1102 1 @ 0x48e7cd 0x938ca5 0x9352bb 0x9363e5 0x90f4c2 0x921da2 0x471e01 # 0x48e7cc sync.(*RWMutex).Unlock+0xac /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/sync/rwmutex.go:198 # 0x938ca4 github.com/couchbase/goxdcr/parts.(*Router).newWrappedMCRequest+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/router.go:1977 # 0x9352ba github.com/couchbase/goxdcr/parts.(*Router).ComposeMCRequest+0x3a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/router.go:1460 # 0x9363e4 github.com/couchbase/goxdcr/parts.(*Router).Route+0x3e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/router.go:1663 # 0x90f4c1 github.com/couchbase/goxdcr/connector.(*Router).Forward+0x141 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/connector/router.go:113 # 0x921da1 github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0xcc1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1102 1 @ 0x4c33a7 0x4c14a5 0x4de76e 0x4de756 0x4de4d6 0x4e8cbe 0x4e8cb6 0x69315f 0x69392c 0xdfc0ca 0x471e01 # 0x4c33a6 syscall.Syscall+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/syscall/syscall_linux.go:69 # 0x4c14a4 syscall.read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/syscall/zsyscall_linux_amd64.go:711 # 0x4de76d syscall.Read+0x2cd /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/syscall/syscall_unix.go:178 # 0x4de755 internal/poll.ignoringEINTRIO+0x2b5 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:794 # 0x4de4d5 internal/poll.(*FD).Read+0x35 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:163 # 0x4e8cbd os.(*File).read+0x5d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file_posix.go:31 # 0x4e8cb5 os.(*File).Read+0x55 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file.go:118 # 0x69315e bufio.(*Reader).fill+0xfe /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/bufio/bufio.go:106 # 0x69392b bufio.(*Reader).ReadByte+0x2b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/bufio/bufio.go:265 # 0xdfc0c9 github.com/couchbase/goxdcr/replication_manager.pollStdin+0x149 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/replication_manager/replication_manager.go:1039 1 @ 0x4c33a7 0x4c14a5 0x4de76e 0x4de756 0x4de4d6 0x5d7749 0x5e78a5 0x4bb29a 0x68a812 0x68a7ef 0x68b6bd 0x74ab8d 0x471e01 # 0x4c33a6 syscall.Syscall+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/syscall/syscall_linux.go:69 # 0x4c14a4 syscall.read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/syscall/zsyscall_linux_amd64.go:711 # 0x4de76d syscall.Read+0x2cd /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/syscall/syscall_unix.go:178 # 0x4de755 internal/poll.ignoringEINTRIO+0x2b5 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:794 # 0x4de4d5 internal/poll.(*FD).Read+0x35 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:163 # 0x5d7748 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e78a4 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x4bb299 io.ReadAtLeast+0x99 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:332 # 0x68a811 io.ReadFull+0x91 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:351 # 0x68a7ee github.com/couchbase/gomemcached.(*MCRequest).populateRegularBody+0x6e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_req.go:498 # 0x68b6bc github.com/couchbase/gomemcached.(*MCRequest).Receive+0x27c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_req.go:654 # 0x74ab8c github.com/couchbase/gomemcached/client.(*UprFeed).runFeed+0x14c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/upr_feed.go:904 1 @ 0x4c33a7 0x4c14a5 0x4de76e 0x4de756 0x4de4d6 0x5d7749 0x5e78a5 0x4bb29a 0x68c3d9 0x68c3aa 0x745185 0x74514d 0x743176 0x8e05f5 0xd9e5e3 0xda2771 0xda0ca5 0x471e01 # 0x4c33a6 syscall.Syscall+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/syscall/syscall_linux.go:69 # 0x4c14a4 syscall.read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/syscall/zsyscall_linux_amd64.go:711 # 0x4de76d syscall.Read+0x2cd /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/syscall/syscall_unix.go:178 # 0x4de755 internal/poll.ignoringEINTRIO+0x2b5 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:794 # 0x4de4d5 internal/poll.(*FD).Read+0x35 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:163 # 0x5d7748 net.(*netFD).Read+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:55 # 0x5e78a4 net.(*conn).Read+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:183 # 0x4bb299 io.ReadAtLeast+0x99 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:332 # 0x68c3d8 io.ReadFull+0x338 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/io/io.go:351 # 0x68c3a9 github.com/couchbase/gomemcached.(*MCResponse).ReceiveWithBuf+0x309 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:204 # 0x745184 github.com/couchbase/gomemcached.(*MCResponse).Receive+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:156 # 0x74514c github.com/couchbase/gomemcached/client.getResponse+0x6c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/transport.go:35 # 0x743175 github.com/couchbase/gomemcached/client.(*Client).StatsMapForSpecifiedStats+0x1b5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:1461 # 0x8e05f4 github.com/couchbase/goxdcr/utils.(*Utilities).GetHighSeqNos+0x154 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:3016 # 0xd9e5e2 github.com/couchbase/goxdcr/service_impl.(*BucketTopologyService).getHighSeqnosUpdater.func1+0x942 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:814 # 0xda2770 github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).updateOnce+0x570 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1362 # 0xda0ca4 github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).run.func2+0x1e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1176 1 @ 0x4c33a7 0x4c1665 0x4e0a05 0x4e09f7 0x4e0840 0x4e9465 0x4e945b 0x50ef6a 0x50f19a 0x68f97f 0x68f2bf 0x68f597 0x68e72e 0x74715e 0x74afbb 0x471e01 # 0x4c33a6 syscall.Syscall+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/syscall/syscall_linux.go:69 # 0x4c1664 syscall.write+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/syscall/zsyscall_linux_amd64.go:939 # 0x4e0a04 syscall.Write+0x384 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/syscall/syscall_unix.go:206 # 0x4e09f6 internal/poll.ignoringEINTRIO+0x376 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:794 # 0x4e083f internal/poll.(*FD).Write+0x1bf /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:383 # 0x4e9464 os.(*File).write+0x64 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file_posix.go:48 # 0x4e945a os.(*File).Write+0x5a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/os/file.go:175 # 0x50ef69 log.(*Logger).Output+0x349 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:194 # 0x50f199 log.(*Logger).Print+0x59 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/log/log.go:213 # 0x68f97e github.com/couchbase/goutils/logging.(*goLogger).log+0xbe /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goutils/logging/logger_golog.go:157 # 0x68f2be github.com/couchbase/goutils/logging.(*goLogger).Logf+0x5e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goutils/logging/logger_golog.go:106 # 0x68f596 github.com/couchbase/goutils/logging.(*goLogger).Infof+0x36 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goutils/logging/logger_golog.go:128 # 0x68e72d github.com/couchbase/goutils/logging.Infof+0xcd /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goutils/logging/logger.go:291 # 0x74715d github.com/couchbase/gomemcached/client.(*vbStreamNegotiator).handleStreamRequest+0x3fd /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/upr_feed.go:242 # 0x74afba github.com/couchbase/gomemcached/client.(*UprFeed).runFeed+0x57a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/upr_feed.go:933 1 @ 0x4c33a7 0x4c1665 0x4e0a05 0x4e09f7 0x4e0840 0x5d7fc9 0x5e7a85 0x68a495 0x7452c5 0x73cd15 0x7475c5 0x471e01 # 0x4c33a6 syscall.Syscall+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/syscall/syscall_linux.go:69 # 0x4c1664 syscall.write+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/syscall/zsyscall_linux_amd64.go:939 # 0x4e0a04 syscall.Write+0x384 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/syscall/syscall_unix.go:206 # 0x4e09f6 internal/poll.ignoringEINTRIO+0x376 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:794 # 0x4e083f internal/poll.(*FD).Write+0x1bf /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:383 # 0x5d7fc8 net.(*netFD).Write+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:96 # 0x5e7a84 net.(*conn).Write+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:195 # 0x68a494 github.com/couchbase/gomemcached.(*MCRequest).Transmit+0x194 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_req.go:447 # 0x7452c4 github.com/couchbase/gomemcached/client.transmitRequest+0x44 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/transport.go:54 # 0x73cd14 github.com/couchbase/gomemcached/client.(*Client).Transmit+0xb4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:290 # 0x7475c4 github.com/couchbase/gomemcached/client.(*UprFeed).sendCommands+0xe4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/upr_feed.go:348 1 @ 0x4c33a7 0x4c1665 0x4e0a05 0x4e09f7 0x4e0840 0x5d7fc9 0x5e7a85 0x9530c2 0x952689 0x941525 0x9401d3 0x941648 0x93f449 0x93f025 0x471e01 # 0x4c33a6 syscall.Syscall+0x26 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/syscall/syscall_linux.go:69 # 0x4c1664 syscall.write+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/syscall/zsyscall_linux_amd64.go:939 # 0x4e0a04 syscall.Write+0x384 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/syscall/syscall_unix.go:206 # 0x4e09f6 internal/poll.ignoringEINTRIO+0x376 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:794 # 0x4e083f internal/poll.(*FD).Write+0x1bf /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_unix.go:383 # 0x5d7fc8 net.(*netFD).Write+0x28 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:96 # 0x5e7a84 net.(*conn).Write+0x44 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:195 # 0x9530c1 github.com/couchbase/goxdcr/parts.(*XmemNozzle).writeToClientWithoutThrottling+0xc1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3463 # 0x952688 github.com/couchbase/goxdcr/parts.(*XmemNozzle).writeToClient+0x388 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3340 # 0x941524 github.com/couchbase/goxdcr/parts.(*XmemNozzle).sendWithRetry+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1434 # 0x9401d2 github.com/couchbase/goxdcr/parts.(*XmemNozzle).batchSetMetaWithRetry+0x892 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1218 # 0x941647 github.com/couchbase/goxdcr/parts.(*XmemNozzle).sendSetMeta_internal+0x47 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1448 # 0x93f448 github.com/couchbase/goxdcr/parts.(*XmemNozzle).processBatch+0x48 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1121 # 0x93f024 github.com/couchbase/goxdcr/parts.(*XmemNozzle).processData_sendbatch+0x4e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1077 1 @ 0x4cd57a 0x7e71ef 0x927b09 0x471e01 # 0x4cd579 time.NewTicker+0x119 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/time/tick.go:22 # 0x7e71ee github.com/couchbase/goxdcr/base.ExecWithTimeout+0x10e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/simple_utils.go:53 # 0x927b08 github.com/couchbase/goxdcr/parts.(*DcpNozzle).checkInactiveUprStreams+0x1a8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1801 1 @ 0x4cf0aa 0x4cf154 0x921dd9 0x471e01 # 0x4cf0a9 time.Time.Sub+0x1c9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/time/time.go:884 # 0x4cf153 time.Since+0x73 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/time/time.go:919 # 0x921dd8 github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0xcf8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1109 1 @ 0x4cf0aa 0x4cf154 0x94e035 0x471e01 # 0x4cf0a9 time.Time.Sub+0x1c9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/time/time.go:884 # 0x4cf153 time.Since+0x73 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/time/time.go:919 # 0x94e034 github.com/couchbase/goxdcr/parts.(*XmemNozzle).receiveResponse+0x17d4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2772 1 @ 0x4cf0aa 0x4cf154 0x94e058 0x471e01 # 0x4cf0a9 time.Time.Sub+0x1c9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/time/time.go:884 # 0x4cf153 time.Since+0x73 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/time/time.go:919 # 0x94e057 github.com/couchbase/goxdcr/parts.(*XmemNozzle).receiveResponse+0x17f7 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2773 1 @ 0x4cf0aa 0x4cf214 0x4dd648 0x5e80c8 0x5e80bb 0x5e80c9 0x8001c4 0x7ffe90 0x9522bb 0x953091 0x952689 0x94a32c 0x950c4b 0x939f54 0x9506c9 0x471e01 # 0x4cf0a9 time.Time.Sub+0x1c9 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/time/time.go:884 # 0x4cf213 time.Until+0x73 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/time/time.go:932 # 0x4dd647 internal/poll.setDeadlineImpl+0xa7 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:149 # 0x5e80c7 internal/poll.(*FD).SetWriteDeadline+0x47 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/internal/poll/fd_poll_runtime.go:143 # 0x5e80ba net.(*netFD).SetWriteDeadline+0x3a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/fd_posix.go:146 # 0x5e80c8 net.(*conn).SetWriteDeadline+0x48 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/net/net.go:261 # 0x8001c3 github.com/couchbase/goxdcr/base.(*XmemClient).setWriteTimeout+0x83 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/xmem_client.go:161 # 0x7ffe8f github.com/couchbase/goxdcr/base.(*XmemClient).GetConn+0x12f /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/base/xmem_client.go:149 # 0x9522ba github.com/couchbase/goxdcr/parts.(*XmemNozzle).getConn+0x9a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3298 # 0x953090 github.com/couchbase/goxdcr/parts.(*XmemNozzle).writeToClientWithoutThrottling+0x90 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3458 # 0x952688 github.com/couchbase/goxdcr/parts.(*XmemNozzle).writeToClient+0x388 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3340 # 0x94a32b github.com/couchbase/goxdcr/parts.(*XmemNozzle).sendSingleSetMeta+0xab /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2334 # 0x950c4a github.com/couchbase/goxdcr/parts.(*XmemNozzle).resendIfTimeout+0x46a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3051 # 0x939f53 github.com/couchbase/goxdcr/parts.(*requestBuffer).modSlot+0x53 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:296 # 0x9506c8 github.com/couchbase/goxdcr/parts.(*XmemNozzle).checkAndRepairBufferMonitor+0x348 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2995 1 @ 0x4cf6c5 0x9358d3 0x9363e5 0x90f4c2 0x921da2 0x471e01 # 0x4cf6c4 time.Now+0x84 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.20.6/go/src/time/time.go:1110 # 0x9358d2 github.com/couchbase/goxdcr/parts.(*Router).ComposeMCRequest+0x652 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/router.go:1546 # 0x9363e4 github.com/couchbase/goxdcr/parts.(*Router).Route+0x3e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/router.go:1663 # 0x90f4c1 github.com/couchbase/goxdcr/connector.(*Router).Forward+0x141 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/connector/router.go:113 # 0x921da1 github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0xcc1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1102 1 @ 0x68c610 0x745185 0x74514d 0x73d0f3 0x9534ab 0x94c9c5 0x471e01 # 0x68c60f github.com/couchbase/gomemcached.(*MCResponse).ReceiveWithBuf+0x56f /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:162 # 0x745184 github.com/couchbase/gomemcached.(*MCResponse).Receive+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/mc_res.go:156 # 0x74514c github.com/couchbase/gomemcached/client.getResponse+0x6c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/transport.go:35 # 0x73d0f2 github.com/couchbase/gomemcached/client.(*Client).Receive+0x52 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/gomemcached/client/mc.go:333 # 0x9534aa github.com/couchbase/goxdcr/parts.(*XmemNozzle).readFromClient+0x18a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3498 # 0x94c9c4 github.com/couchbase/goxdcr/parts.(*XmemNozzle).receiveResponse+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2618 1 @ 0x8c7a93 0x923d54 0x921d69 0x471e01 # 0x8c7a92 github.com/couchbase/goxdcr/utils.(*WrappedUprPool).Get+0x112 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/dataObjPools.go:42 # 0x923d53 github.com/couchbase/goxdcr/parts.(*DcpNozzle).composeWrappedUprEvent+0x733 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1290 # 0x921d68 github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0xc88 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1095 1 @ 0x8e1f4b 0xd9aefa 0xda2771 0xda0ca5 0x471e01 # 0x8e1f4a github.com/couchbase/goxdcr/utils.(*Utilities).GetReplicasInfo+0xf8a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:3282 # 0xd9aef9 github.com/couchbase/goxdcr/service_impl.(*BucketTopologyService).getOrCreateRemoteWatcher.func1+0x2d9 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:494 # 0xda2770 github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).updateOnce+0x570 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1362 # 0xda0ca4 github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).run.func2+0x1e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1176 1 @ 0x8ed221 0x471e01 # 0x8ed220 github.com/couchbase/goxdcr/peerToPeer.(*P2PManagerImpl).sendToSpecifiedPeersOnce.func1.1+0x0 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/peerToPeer/peerToPeerManager.go:288 1 @ 0x90e61e 0x90e03d 0x921d09 0x471e01 # 0x90e61d github.com/couchbase/goxdcr/component.(*AsyncComponentEventListenerImpl).OnEvent+0x7d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/component/async_listener.go:51 # 0x90e03c github.com/couchbase/goxdcr/component.(*AbstractComponent).RaiseEvent+0xdc /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/component/abstract_component.go:92 # 0x921d08 github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0xc28 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1093 1 @ 0x91060f 0x95246e 0x952487 0x94a32c 0x950c4b 0x939f54 0x9506c9 0x471e01 # 0x91060e github.com/couchbase/goxdcr/parts.(*AbstractPart).State+0xce /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/abstract_part.go:68 # 0x95246d github.com/couchbase/goxdcr/parts.(*XmemNozzle).validateRunningState+0x16d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3302 # 0x952486 github.com/couchbase/goxdcr/parts.(*XmemNozzle).writeToClient+0x186 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3320 # 0x94a32b github.com/couchbase/goxdcr/parts.(*XmemNozzle).sendSingleSetMeta+0xab /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2334 # 0x950c4a github.com/couchbase/goxdcr/parts.(*XmemNozzle).resendIfTimeout+0x46a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:3051 # 0x939f53 github.com/couchbase/goxdcr/parts.(*requestBuffer).modSlot+0x53 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:296 # 0x9506c8 github.com/couchbase/goxdcr/parts.(*XmemNozzle).checkAndRepairBufferMonitor+0x348 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:2995 1 @ 0x927702 0x925f3f 0x924805 0x9247ec 0x471e01 # 0x927701 github.com/couchbase/goxdcr/parts.(*DcpNozzle).GetStreamState+0x101 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1767 # 0x925f3e github.com/couchbase/goxdcr/parts.(*DcpNozzle).getDcpStreams+0x9e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1562 # 0x924804 github.com/couchbase/goxdcr/parts.(*DcpNozzle).nonInitDcpStreams+0x284 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1587 # 0x9247eb github.com/couchbase/goxdcr/parts.(*DcpNozzle).startUprStreams+0x26b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1355 1 @ 0x937d4e 0x93723a 0x936649 0x90f4c2 0x921da2 0x471e01 # 0x937d4d github.com/couchbase/goxdcr/parts.(*Router).prepareMcRequest+0x9ed /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/router.go:1760 # 0x937239 github.com/couchbase/goxdcr/parts.(*Router).RouteCollection+0x4d9 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/router.go:1748 # 0x936648 github.com/couchbase/goxdcr/parts.(*Router).Route+0x648 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/router.go:1679 # 0x90f4c1 github.com/couchbase/goxdcr/connector.(*Router).Forward+0x141 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/connector/router.go:113 # 0x921da1 github.com/couchbase/goxdcr/parts.(*DcpNozzle).processData+0xcc1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/dcp_nozzle.go:1102 1 @ 0x93a7a6 0x940030 0x941648 0x93f449 0x93f025 0x471e01 # 0x93a7a5 github.com/couchbase/goxdcr/parts.(*requestBuffer).enSlot+0x345 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:395 # 0x94002f github.com/couchbase/goxdcr/parts.(*XmemNozzle).batchSetMetaWithRetry+0x6ef /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1202 # 0x941647 github.com/couchbase/goxdcr/parts.(*XmemNozzle).sendSetMeta_internal+0x47 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1448 # 0x93f448 github.com/couchbase/goxdcr/parts.(*XmemNozzle).processBatch+0x48 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1121 # 0x93f024 github.com/couchbase/goxdcr/parts.(*XmemNozzle).processData_sendbatch+0x4e4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/parts/xmem_nozzle.go:1077 1 @ 0xa3b941 0x471e01 # 0xa3b940 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).scheduleFutureRefresh.func1+0x0 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:2214 1 @ 0xdbc959 0x90e8f8 0x471e01 # 0xdbc958 github.com/couchbase/goxdcr/service_impl.(*ThroughSeqnoTrackerSvc).ProcessEvent+0x16d8 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/through_seqno_tracker_service.go:873 # 0x90e8f7 github.com/couchbase/goxdcr/component.(*AsyncComponentEventListenerImpl).run+0x157 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/component/async_listener.go:74 1 @ 0xdbe2b4 0xdc04a5 0x9ea326 0x9ea9d4 0x471e01 # 0xdbe2b3 github.com/couchbase/goxdcr/service_impl.(*ThroughSeqnoTrackerSvc).setTargetSeqnoManifestIdPair+0x193 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/through_seqno_tracker_service.go:1267 # 0xdc04a4 github.com/couchbase/goxdcr/service_impl.(*ThroughSeqnoTrackerSvc).SetStartSeqno+0xa4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/service_impl/through_seqno_tracker_service.go:1592 # 0x9ea325 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).setTimestampForVB+0x245 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1289 # 0x9ea9d3 github.com/couchbase/goxdcr/pipeline_svc.(*CheckpointManager).startSeqnoGetter+0x353 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/goxdcr/pipeline_svc/checkpoint_manager.go:1335