goroutine profile: total 122 17 @ 0x10003cab6 0x1000082a5 0x100007e5d 0x1005b48f2 0x1003c4c73 0x10006cfe1 # 0x1005b48f1 github.com/couchbase/goxdcr/adminport.(*httpServer).systemHandler.func2+0xb1 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/adminport/admin_httpd.go:136 # 0x1003c4c72 github.com/couchbase/goxdcr/base.ExecWithTimeout3.func1+0x32 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/base/simple_utils.go:115 11 @ 0x10003cab6 0x10004c3b2 0x1003038f5 0x10006cfe1 # 0x1003038f4 net/http.(*persistConn).writeLoop+0xf4 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:2392 9 @ 0x10003cab6 0x10004c3b2 0x1003027f9 0x10006cfe1 # 0x1003027f8 net/http.(*persistConn).readLoop+0xd98 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:2213 7 @ 0x10003cab6 0x100036097 0x1000671e9 0x1000c9872 0x1000cabda 0x1000cabc8 0x1001cfec9 0x1001df9a5 0x100300dee 0x100153dc3 0x1001549af 0x10029ece5 0x10029e768 0x10029ea2e 0x1002f595c 0x1002f5825 0x100305522 0x100264f9f 0x100264b9b 0x1002647f8 0x1005597eb 0x10006cfe1 # 0x1000671e8 internal/poll.runtime_pollWait+0x88 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/runtime/netpoll.go:302 # 0x1000c9871 internal/poll.(*pollDesc).wait+0x31 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_poll_runtime.go:83 # 0x1000cabd9 internal/poll.(*pollDesc).waitRead+0x259 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_poll_runtime.go:88 # 0x1000cabc7 internal/poll.(*FD).Read+0x247 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_unix.go:167 # 0x1001cfec8 net.(*netFD).Read+0x28 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/fd_posix.go:55 # 0x1001df9a4 net.(*conn).Read+0x44 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/net.go:183 # 0x100300ded net/http.(*persistConn).Read+0x4d /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:1929 # 0x100153dc2 bufio.(*Reader).fill+0x102 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/bufio/bufio.go:106 # 0x1001549ae bufio.(*Reader).ReadSlice+0x2e /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/bufio/bufio.go:371 # 0x10029ece4 net/http/internal.readChunkLine+0x24 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/internal/chunked.go:129 # 0x10029e767 net/http/internal.(*chunkedReader).beginChunk+0x27 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/internal/chunked.go:48 # 0x10029ea2d net/http/internal.(*chunkedReader).Read+0x14d /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/internal/chunked.go:98 # 0x1002f595b net/http.(*body).readLocked+0x3b /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transfer.go:844 # 0x1002f5824 net/http.(*body).Read+0x124 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transfer.go:836 # 0x100305521 net/http.(*bodyEOFSignal).Read+0x141 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:2774 # 0x100264f9e encoding/json.(*Decoder).refill+0x17e /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/encoding/json/stream.go:165 # 0x100264b9a encoding/json.(*Decoder).readValue+0xba /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/encoding/json/stream.go:140 # 0x1002647f7 encoding/json.(*Decoder).Decode+0x77 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/encoding/json/stream.go:63 # 0x1005597ea github.com/couchbase/cbauth/metakv.doRunObserveChildren.func1+0x10a /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:284 5 @ 0x10003cab6 0x10004c3b2 0x1003c497c 0x1005b40e5 0x1005b4c9d 0x1002eab69 0x1002ec7bb 0x1002e7c77 0x10006cfe1 # 0x1003c497b github.com/couchbase/goxdcr/base.ExecWithTimeout3+0x29b /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/base/simple_utils.go:124 # 0x1005b40e4 github.com/couchbase/goxdcr/adminport.(*httpServer).systemHandler+0x1c4 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/adminport/admin_httpd.go:131 # 0x1005b4c9c github.com/couchbase/goxdcr/adminport.(*Handler).ServeHTTP+0x13c /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/adminport/admin_httpd.go:204 # 0x1002eab68 net/http.(*ServeMux).ServeHTTP+0x148 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/server.go:2462 # 0x1002ec7ba net/http.serverHandler.ServeHTTP+0x43a /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/server.go:2916 # 0x1002e7c76 net/http.(*conn).serve+0x5d6 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/server.go:1966 5 @ 0x10003cab6 0x10004c3b2 0x100a00630 0x10006cfe1 # 0x100a0062f github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).run.func2+0x18f /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1231 4 @ 0x10003cab6 0x10000914c 0x100008b78 0x100a13c6e 0x100a13be5 0x10006cfe1 # 0x100a13c6d github.com/couchbase/goxdcr/service_impl.(*ResolverSvc).resolveOne+0x4d /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/service_impl/resolver_service.go:162 # 0x100a13be4 github.com/couchbase/goxdcr/service_impl.(*ResolverSvc).resolverWorker+0x24 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/service_impl/resolver_service.go:157 4 @ 0x10003cab6 0x100036097 0x1000671e9 0x1000c9872 0x1000cabda 0x1000cabc8 0x1001cfec9 0x1001df9a5 0x1002e1b5f 0x10006cfe1 # 0x1000671e8 internal/poll.runtime_pollWait+0x88 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/runtime/netpoll.go:302 # 0x1000c9871 internal/poll.(*pollDesc).wait+0x31 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_poll_runtime.go:83 # 0x1000cabd9 internal/poll.(*pollDesc).waitRead+0x259 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_poll_runtime.go:88 # 0x1000cabc7 internal/poll.(*FD).Read+0x247 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_unix.go:167 # 0x1001cfec8 net.(*netFD).Read+0x28 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/fd_posix.go:55 # 0x1001df9a4 net.(*conn).Read+0x44 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/net.go:183 # 0x1002e1b5e net/http.(*connReader).backgroundRead+0x3e /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/server.go:672 4 @ 0x10003cab6 0x10004c3b2 0x100559349 0x1005598f8 0x1005598ba 0x100a43468 0x10006cfe1 # 0x100559348 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x428 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:301 # 0x1005598f7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:259 # 0x1005598b9 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x19 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:389 # 0x100a43467 github.com/couchbase/goxdcr/replication_manager.(*MetakvChangeListener).observeChildren+0x87 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/replication_manager/metakv_change_listener.go:79 3 @ 0x10003cab6 0x10004c3b2 0x1004e21bf 0x10006cfe1 # 0x1004e21be github.com/couchbase/goxdcr/peerToPeer.(*HandlerCommon).ClearOpaqueBg+0x7e /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/peerToPeer/handlerCommon.go:148 3 @ 0x10003cab6 0x10004c3b2 0x100559349 0x1005598f8 0x1005598ba 0x100663316 0x1006613c2 0x10066325a 0x10006cfe1 # 0x100559348 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x428 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:301 # 0x1005598f7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:259 # 0x1005598b9 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x19 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/cbauth/metakv/metakv.go:389 # 0x100663315 github.com/couchbase/eventing-ee/evaluator/impl/storagedriver.(*MkvStoreDriver).ObserveChanges.func1+0x55 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/eventing-ee/evaluator/impl/storagedriver/metakv_driver.go:247 # 0x1006613c1 github.com/couchbase/eventing-ee/evaluator/utils.ExponentialBackoffRetry+0x41 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/eventing-ee/evaluator/utils/retry.go:57 # 0x100663259 github.com/couchbase/eventing-ee/evaluator/impl/storagedriver.(*MkvStoreDriver).ObserveChanges+0x99 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/eventing-ee/evaluator/impl/storagedriver/metakv_driver.go:245 2 @ 0x10003cab6 0x10000914c 0x100008b78 0x100a00331 0x10006cfe1 # 0x100a00330 github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).run+0x350 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1248 2 @ 0x10003cab6 0x100036097 0x1000671e9 0x1000c9872 0x1000cabda 0x1000cabc8 0x1001cfec9 0x1001df9a5 0x100300dee 0x100153dc3 0x100153f1d 0x100301c0c 0x10006cfe1 # 0x1000671e8 internal/poll.runtime_pollWait+0x88 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/runtime/netpoll.go:302 # 0x1000c9871 internal/poll.(*pollDesc).wait+0x31 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_poll_runtime.go:83 # 0x1000cabd9 internal/poll.(*pollDesc).waitRead+0x259 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_poll_runtime.go:88 # 0x1000cabc7 internal/poll.(*FD).Read+0x247 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_unix.go:167 # 0x1001cfec8 net.(*netFD).Read+0x28 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/fd_posix.go:55 # 0x1001df9a4 net.(*conn).Read+0x44 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/net.go:183 # 0x100300ded net/http.(*persistConn).Read+0x4d /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:1929 # 0x100153dc2 bufio.(*Reader).fill+0x102 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/bufio/bufio.go:106 # 0x100153f1c bufio.(*Reader).Peek+0x5c /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/bufio/bufio.go:144 # 0x100301c0b net/http.(*persistConn).readLoop+0x1ab /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:2093 2 @ 0x10003cab6 0x100036097 0x1000671e9 0x1000c9872 0x1000cabda 0x1000cabc8 0x1001cfec9 0x1001df9a5 0x100300dee 0x100153dc3 0x1001549af 0x10029ece5 0x10029e768 0x10029ea2e 0x1002f595c 0x1002f5825 0x100305522 0x100153dc3 0x1001549af 0x100154df4 0x100154f9d 0x100507e5d 0x10006cfe1 # 0x1000671e8 internal/poll.runtime_pollWait+0x88 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/runtime/netpoll.go:302 # 0x1000c9871 internal/poll.(*pollDesc).wait+0x31 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_poll_runtime.go:83 # 0x1000cabd9 internal/poll.(*pollDesc).waitRead+0x259 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_poll_runtime.go:88 # 0x1000cabc7 internal/poll.(*FD).Read+0x247 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_unix.go:167 # 0x1001cfec8 net.(*netFD).Read+0x28 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/fd_posix.go:55 # 0x1001df9a4 net.(*conn).Read+0x44 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/net.go:183 # 0x100300ded net/http.(*persistConn).Read+0x4d /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:1929 # 0x100153dc2 bufio.(*Reader).fill+0x102 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/bufio/bufio.go:106 # 0x1001549ae bufio.(*Reader).ReadSlice+0x2e /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/bufio/bufio.go:371 # 0x10029ece4 net/http/internal.readChunkLine+0x24 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/internal/chunked.go:129 # 0x10029e767 net/http/internal.(*chunkedReader).beginChunk+0x27 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/internal/chunked.go:48 # 0x10029ea2d net/http/internal.(*chunkedReader).Read+0x14d /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/internal/chunked.go:98 # 0x1002f595b net/http.(*body).readLocked+0x3b /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transfer.go:844 # 0x1002f5824 net/http.(*body).Read+0x124 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transfer.go:836 # 0x100305521 net/http.(*bodyEOFSignal).Read+0x141 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/transport.go:2774 # 0x100153dc2 bufio.(*Reader).fill+0x102 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/bufio/bufio.go:106 # 0x1001549ae bufio.(*Reader).ReadSlice+0x2e /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/bufio/bufio.go:371 # 0x100154df3 bufio.(*Reader).collectFragments+0x73 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/bufio/bufio.go:446 # 0x100154f9c bufio.(*Reader).ReadBytes+0x1c /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/bufio/bufio.go:474 # 0x100507e5c github.com/couchbase/goxdcr/streamApiWatcher.(*StreamApiWatcherImpl).runStreamingEndpoint.func2+0xbc /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/streamApiWatcher/streamApiWatcher.go:222 2 @ 0x10003cab6 0x10004c3b2 0x100506925 0x10006cfe1 # 0x100506924 github.com/couchbase/goxdcr/streamApiWatcher.(*StreamApiWatcherImpl).watchClusterChanges+0x204 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/streamApiWatcher/streamApiWatcher.go:134 2 @ 0x10003cab6 0x10004c3b2 0x100507a45 0x100506d79 0x10006cfe1 # 0x100507a44 github.com/couchbase/goxdcr/streamApiWatcher.(*StreamApiWatcherImpl).runStreamingEndpoint+0xc04 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/streamApiWatcher/streamApiWatcher.go:245 # 0x100506d78 github.com/couchbase/goxdcr/streamApiWatcher.(*StreamApiWatcherImpl).runStreamingQuery+0x58 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/streamApiWatcher/streamApiWatcher.go:163 2 @ 0x10003cab6 0x10004c3b2 0x10056667d 0x10006cfe1 # 0x10056667c github.com/couchbase/goxdcr/metadata_svc.(*CheckpointsServiceCacheImpl).Run+0x17c /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/metadata_svc/checkpointsServiceCache.go:218 1 @ 0x10003cab6 0x1000082a5 0x100007e5d 0x100a39efe 0x100a39fcb 0x10006cfe1 # 0x100a39efd github.com/couchbase/goxdcr/gen_server.(*GenServer).SendMsg_async+0x33d /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/gen_server/gen_server.go:213 # 0x100a39fca github.com/couchbase/goxdcr/replication_manager.(*Adminport).Start+0x40a /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/replication_manager/adminport.go:133 1 @ 0x10003cab6 0x10000914c 0x100008b78 0x100653005 0x10064b37f 0x10064b34a 0x1004d67aa 0x10064b1cd 0x100645df8 0x1004d64db 0x1004d629b 0x100a45786 0x1006380f2 0x100634a78 0x100632d7d 0x100083ca2 0x100632787 0x100632755 0x10006cfe1 # 0x100653004 github.com/couchbase/goxdcr/backfill_manager.(*BackfillRequestHandler).handleBackfillRequestWithArgs+0x204 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/backfill_manager/backfill_request_handler.go:400 # 0x10064b37e github.com/couchbase/goxdcr/backfill_manager.(*BackfillRequestHandler).HandleBackfillRequest+0x5e /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/backfill_manager/backfill_request_handler.go:368 # 0x10064b349 github.com/couchbase/goxdcr/backfill_manager.(*BackfillMgr).handleExplicitMapChangeBackfillReq.func1+0x29 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/backfill_manager/backfill_manager.go:1377 # 0x1004d67a9 github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithFinishSignal+0x109 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2596 # 0x10064b1cc github.com/couchbase/goxdcr/backfill_manager.(*BackfillMgr).handleExplicitMapChangeBackfillReq+0x26c /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/backfill_manager/backfill_manager.go:1379 # 0x100645df7 github.com/couchbase/goxdcr/backfill_manager.(*BackfillMgr).GetExplicitMappingChangeHandler.func1+0x437 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/backfill_manager/backfill_manager.go:679 # 0x1004d64da github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutorWithOriginalError+0xba /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2568 # 0x1004d629a github.com/couchbase/goxdcr/utils.(*Utilities).ExponentialBackoffExecutor+0x3a /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/utils/utils.go:2556 # 0x100a45785 github.com/couchbase/goxdcr/replication_manager.needSpecialCallbackUpdate.func1+0x65 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/replication_manager/metakv_change_listener.go:317 # 0x1006380f1 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).executeQueuedCallbacks+0x111 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:2287 # 0x100634a77 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).update+0x557 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1843 # 0x100632d7c github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run.func1+0x5bc /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1545 # 0x100083ca1 sync.(*Once).doSlow+0xc1 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/sync/once.go:68 # 0x100632786 sync.(*Once).Do+0x46 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/sync/once.go:59 # 0x100632754 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).run+0x14 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:1523 1 @ 0x10003cab6 0x10000914c 0x100008b78 0x1006d5ba5 0x10006cfe1 # 0x1006d5ba4 github.com/couchbase/regulator/utils/logging.duplicateFlushLoop+0x24 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/regulator/utils/logging/logger_golog.go:253 1 @ 0x10003cab6 0x10000914c 0x100008b78 0x100a616c5 0x10003c6f2 0x10006cfe1 # 0x100a616c4 main.main+0x1024 goproj/src/github.com/couchbase/goxdcr/main/main.go:280 # 0x10003c6f1 runtime.main+0x211 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/runtime/proc.go:250 1 @ 0x10003cab6 0x100036097 0x1000671e9 0x1000c9872 0x1000cabda 0x1000cabc8 0x1001cfec9 0x1001df9a5 0x100154394 0x1004bcd85 0x100264f9f 0x100264b9b 0x1002647f8 0x1004bbfe5 0x1004ba5c6 0x1004ba11b 0x1004b9667 0x1004bdabc 0x1004be158 0x1004c059d 0x1004c0885 0x10006cfe1 # 0x1000671e8 internal/poll.runtime_pollWait+0x88 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/runtime/netpoll.go:302 # 0x1000c9871 internal/poll.(*pollDesc).wait+0x31 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_poll_runtime.go:83 # 0x1000cabd9 internal/poll.(*pollDesc).waitRead+0x259 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_poll_runtime.go:88 # 0x1000cabc7 internal/poll.(*FD).Read+0x247 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_unix.go:167 # 0x1001cfec8 net.(*netFD).Read+0x28 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/fd_posix.go:55 # 0x1001df9a4 net.(*conn).Read+0x44 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/net.go:183 # 0x100154393 bufio.(*Reader).Read+0x1b3 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/bufio/bufio.go:236 # 0x1004bcd84 github.com/couchbase/cbauth/revrpc.(*minirwc).Read+0x24 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/cbauth/revrpc/revrpc.go:103 # 0x100264f9e encoding/json.(*Decoder).refill+0x17e /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/encoding/json/stream.go:165 # 0x100264b9a encoding/json.(*Decoder).readValue+0xba /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/encoding/json/stream.go:140 # 0x1002647f7 encoding/json.(*Decoder).Decode+0x77 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/encoding/json/stream.go:63 # 0x1004bbfe4 net/rpc/jsonrpc.(*serverCodec).ReadRequestHeader+0x84 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/rpc/jsonrpc/server.go:66 # 0x1004ba5c5 net/rpc.(*Server).readRequestHeader+0x65 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/rpc/server.go:587 # 0x1004ba11a net/rpc.(*Server).readRequest+0x3a /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/rpc/server.go:547 # 0x1004b9666 net/rpc.(*Server).ServeCodec+0x86 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/rpc/server.go:462 # 0x1004bdabb github.com/couchbase/cbauth/revrpc.(*Service).Run+0x77b /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/cbauth/revrpc/revrpc.go:236 # 0x1004be157 github.com/couchbase/cbauth/revrpc.BabysitService+0x57 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/cbauth/revrpc/revrpc.go:335 # 0x1004c059c github.com/couchbase/cbauth.runRPCForSvc+0xbc /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/cbauth/default.go:57 # 0x1004c0884 github.com/couchbase/cbauth.startDefault.func1+0x24 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/cbauth/default.go:66 1 @ 0x10003cab6 0x100036097 0x1000671e9 0x1000c9872 0x1000cea8c 0x1000cea79 0x1001d1fd5 0x1001e6b48 0x1001e5d3d 0x1002ecd25 0x1002ec95d 0x1005b3b76 0x10006cfe1 # 0x1000671e8 internal/poll.runtime_pollWait+0x88 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/runtime/netpoll.go:302 # 0x1000c9871 internal/poll.(*pollDesc).wait+0x31 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_poll_runtime.go:83 # 0x1000cea8b internal/poll.(*pollDesc).waitRead+0x22b /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_poll_runtime.go:88 # 0x1000cea78 internal/poll.(*FD).Accept+0x218 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_unix.go:614 # 0x1001d1fd4 net.(*netFD).accept+0x34 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/fd_unix.go:172 # 0x1001e6b47 net.(*TCPListener).accept+0x27 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/tcpsock_posix.go:139 # 0x1001e5d3c net.(*TCPListener).Accept+0x3c /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/tcpsock.go:288 # 0x1002ecd24 net/http.(*Server).Serve+0x384 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/server.go:3039 # 0x1002ec95c net/http.(*Server).ListenAndServe+0x7c /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/server.go:2968 # 0x1005b3b75 github.com/couchbase/goxdcr/adminport.(*httpServer).Start.func1+0xf5 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/adminport/admin_httpd.go:93 1 @ 0x10003cab6 0x10004c3b2 0x1004566e5 0x10006cfe1 # 0x1004566e4 github.com/couchbase/cbauth/cbauthimpl.(*cfgChangeNotifier).loop+0x84 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/cbauth/cbauthimpl/impl.go:325 1 @ 0x10003cab6 0x10004c3b2 0x100456cc7 0x10006cfe1 # 0x100456cc6 github.com/couchbase/cbauth/cbauthimpl.(*tlsNotifier).loop+0x66 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/cbauth/cbauthimpl/impl.go:405 1 @ 0x10003cab6 0x10004c3b2 0x1004e0b8f 0x10006cfe1 # 0x1004e0b8e github.com/couchbase/goxdcr/peerToPeer.(*DiscoveryHandler).handler+0xae /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/peerToPeer/discoveryHandler.go:55 1 @ 0x10003cab6 0x10004c3b2 0x1004f5650 0x10006cfe1 # 0x1004f564f github.com/couchbase/goxdcr/peerToPeer.(*PeriodicPushHandler).handler.func2+0x8f /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/peerToPeer/periodicPushHandler.go:80 1 @ 0x10003cab6 0x10004c3b2 0x1004f57d9 0x10006cfe1 # 0x1004f57d8 github.com/couchbase/goxdcr/peerToPeer.(*PeriodicPushHandler).handler.func1+0x98 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/peerToPeer/periodicPushHandler.go:62 1 @ 0x10003cab6 0x10004c3b2 0x1004f7e65 0x10006cfe1 # 0x1004f7e64 github.com/couchbase/goxdcr/peerToPeer.(*ReplicaCacheMonitor).Run+0xa4 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/peerToPeer/replicaCache.go:108 1 @ 0x10003cab6 0x10004c3b2 0x1004f97ab 0x10006cfe1 # 0x1004f97aa github.com/couchbase/goxdcr/peerToPeer.(*ReplicaReplicatorImpl).run+0xca /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/peerToPeer/replicaReplicator.go:229 1 @ 0x10003cab6 0x10004c3b2 0x1004fb32f 0x10006cfe1 # 0x1004fb32e github.com/couchbase/goxdcr/peerToPeer.(*ReplicatorAgentImpl).run+0x46e /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/peerToPeer/replicaReplicator.go:508 1 @ 0x10003cab6 0x10004c3b2 0x100501590 0x10006cfe1 # 0x10050158f github.com/couchbase/goxdcr/peerToPeer.(*VBMasterCheckHandler).handler.func2+0x8f /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/peerToPeer/vbMasterCheckHandler.go:345 1 @ 0x10003cab6 0x10004c3b2 0x100501710 0x10006cfe1 # 0x10050170f github.com/couchbase/goxdcr/peerToPeer.(*VBMasterCheckHandler).handler.func1+0x8f /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/peerToPeer/vbMasterCheckHandler.go:329 1 @ 0x10003cab6 0x10004c3b2 0x10057393e 0x10006cfe1 # 0x10057393d github.com/couchbase/goxdcr/metadata_svc.(*CollectionsManifestAgent).runPeriodicRefresh+0xdd /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/metadata_svc/collections_manifest_service.go:527 1 @ 0x10003cab6 0x10004c3b2 0x100573ac5 0x10006cfe1 # 0x100573ac4 github.com/couchbase/goxdcr/metadata_svc.(*CollectionsManifestAgent).runPersistRequestHandler+0xc4 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/metadata_svc/collections_manifest_service.go:551 1 @ 0x10003cab6 0x10004c3b2 0x10058d249 0x10006cfe1 # 0x10058d248 github.com/couchbase/goxdcr/metadata_svc.(*RemoteClusterAgent).runPeriodicRefresh+0x188 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/metadata_svc/remote_cluster_service.go:1614 1 @ 0x10003cab6 0x10004c3b2 0x1005b58ca 0x10006cfe1 # 0x1005b58c9 github.com/couchbase/goxdcr/gen_server.(*GenServer).run+0xc9 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/gen_server/gen_server.go:83 1 @ 0x10003cab6 0x10004c3b2 0x1005cf265 0x10006cfe1 # 0x1005cf264 github.com/couchbase/goxdcr/supervisor.(*GenericSupervisor).supervising+0x184 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/supervisor/generic_supervisor.go:207 1 @ 0x10003cab6 0x10004c3b2 0x10063ae65 0x10006cfe1 # 0x10063ae64 github.com/couchbase/goxdcr/resource_manager.(*ResourceManager).collectCpuUsage+0x164 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/resource_manager/resource_manager.go:496 1 @ 0x10003cab6 0x10004c3b2 0x10063b185 0x10006cfe1 # 0x10063b184 github.com/couchbase/goxdcr/resource_manager.(*ResourceManager).manageResources+0x164 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/resource_manager/resource_manager.go:514 1 @ 0x10003cab6 0x10004c3b2 0x10063bc45 0x10006cfe1 # 0x10063bc44 github.com/couchbase/goxdcr/resource_manager.(*ResourceManager).logStats+0x164 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/resource_manager/resource_manager.go:599 1 @ 0x10003cab6 0x10004c3b2 0x10064d13d 0x10006cfe1 # 0x10064d13c github.com/couchbase/goxdcr/backfill_manager.(*BackfillMgr).runRetryMonitor+0xdc /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/backfill_manager/backfill_manager.go:1631 1 @ 0x10003cab6 0x10004c3b2 0x1009ec176 0x10006cfe1 # 0x1009ec175 github.com/couchbase/eventing-ee/evaluator/impl.(*SharedStore).periodicScan+0xf5 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/eventing-ee/evaluator/impl/sharedstore.go:391 1 @ 0x10003cab6 0x10004c3b2 0x100a00745 0x10006cfe1 # 0x100a00744 github.com/couchbase/goxdcr/service_impl.(*BucketTopologySvcWatcher).run.func1+0x84 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/service_impl/bucketTopologyService.go:1205 1 @ 0x10003cab6 0x10004c3b2 0x100a22ca5 0x10006cfe1 # 0x100a22ca4 github.com/couchbase/goxdcr/service_impl.(*ThroughputThrottler).update+0x244 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/service_impl/throughput_throttler_service.go:101 1 @ 0x10003cab6 0x10004c3b2 0x100a232c6 0x10006cfe1 # 0x100a232c5 github.com/couchbase/goxdcr/service_impl.(*ThroughputThrottler).clearReassignedTokens+0x245 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/service_impl/throughput_throttler_service.go:187 1 @ 0x10003cab6 0x10004c3b2 0x100a23f5c 0x10006cfe1 # 0x100a23f5b github.com/couchbase/goxdcr/service_impl.(*ThroughputThrottler).logStats+0x1fb /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/service_impl/throughput_throttler_service.go:360 1 @ 0x10003cab6 0x10004c3b2 0x100a560ca 0x10006cfe1 # 0x100a560c9 github.com/couchbase/goxdcr/replication_manager.(*replicationManager).checkReplicationStatus+0x1a9 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/replication_manager/replication_manager.go:442 1 @ 0x10003cab6 0x10004c3b2 0x100a5b385 0x10006cfe1 # 0x100a5b384 github.com/couchbase/goxdcr/replication_manager.logMemStats+0x1c4 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/replication_manager/replication_manager.go:1059 1 @ 0x10003cab6 0x10004d493 0x10004d46d 0x100068865 0x100085392 0x100625987 0x10062910d 0x100644683 0x1005640b3 0x10056325c 0x10056323c 0x10065569e 0x1006515ff 0x10006cfe1 # 0x100068864 sync.runtime_Semacquire+0x24 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/runtime/sema.go:56 # 0x100085391 sync.(*WaitGroup).Wait+0x51 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/sync/waitgroup.go:136 # 0x100625986 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineOpSerializer).StopBackfillWithCb+0x166 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipelineOpSerializer.go:276 # 0x10062910c github.com/couchbase/goxdcr/pipeline_manager.(*PipelineManager).HaltBackfillWithCb+0x2c /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:231 # 0x100644682 github.com/couchbase/goxdcr/backfill_manager.(*BackfillMgr).backfillReplSpecChangeHandlerCallback+0x4a2 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/backfill_manager/backfill_manager.go:548 # 0x1005640b2 github.com/couchbase/goxdcr/metadata_svc.(*BackfillReplicationService).updateCacheInternal+0x1f2 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/metadata_svc/backfill_repl_service.go:611 # 0x10056325b github.com/couchbase/goxdcr/metadata_svc.(*BackfillReplicationService).updateCache+0xfb /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/metadata_svc/backfill_repl_service.go:593 # 0x10056323b github.com/couchbase/goxdcr/metadata_svc.(*BackfillReplicationService).SetBackfillReplSpec+0xdb /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/metadata_svc/backfill_repl_service.go:502 # 0x10065569d github.com/couchbase/goxdcr/backfill_manager.(*BackfillRequestHandler).metaKvOp+0x3d /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/backfill_manager/backfill_request_handler.go:774 # 0x1006515fe github.com/couchbase/goxdcr/backfill_manager.(*BackfillRequestHandler).run+0x47e /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/backfill_manager/backfill_request_handler.go:311 1 @ 0x10003cab6 0x10004d493 0x10004d46d 0x100068865 0x100085392 0x100636746 0x10062e73c 0x10062e72d 0x100627356 0x10006cfe1 # 0x100068864 sync.runtime_Semacquire+0x24 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/runtime/sema.go:56 # 0x100085391 sync.(*WaitGroup).Wait+0x51 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/sync/waitgroup.go:136 # 0x100636745 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).sendStopBackfillPipeline+0xc5 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:2080 # 0x10062e73b github.com/couchbase/goxdcr/pipeline_manager.(*PipelineUpdater).stopBackfillPipeline+0x9b /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:2047 # 0x10062e72c github.com/couchbase/goxdcr/pipeline_manager.(*PipelineManager).StopBackfillWithStoppedCb+0x8c /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipeline_manager.go:948 # 0x100627355 github.com/couchbase/goxdcr/pipeline_manager.(*PipelineOpSerializer).handleJobs+0xf15 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/pipeline_manager/pipelineOpSerializer.go:452 1 @ 0x10003cab6 0x10004d493 0x10004d46d 0x100068985 0x100083905 0x100563f4f 0x100563f2d 0x100564530 0x10056450b 0x1005ae54e 0x1005adfed 0x1005ad2bb 0x1005acd35 0x1005acd36 0x100a438bf 0x10006cfe1 # 0x100068984 sync.runtime_SemacquireMutex+0x24 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/runtime/sema.go:71 # 0x100083904 sync.(*Mutex).lockSlow+0x164 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/sync/mutex.go:162 # 0x100563f4e sync.(*Mutex).Lock+0x8e /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/sync/mutex.go:81 # 0x100563f2c github.com/couchbase/goxdcr/metadata_svc.(*BackfillReplicationService).updateCacheInternal+0x6c /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/metadata_svc/backfill_repl_service.go:602 # 0x10056452f github.com/couchbase/goxdcr/metadata_svc.(*BackfillReplicationService).updateCache+0x34f /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/metadata_svc/backfill_repl_service.go:593 # 0x10056450a github.com/couchbase/goxdcr/metadata_svc.(*BackfillReplicationService).ReplicationSpecChangeCallback+0x32a /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/metadata_svc/backfill_repl_service.go:657 # 0x1005ae54d github.com/couchbase/goxdcr/metadata_svc.(*ReplicationSpecService).executeCallbackWithPriority+0x18d /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/metadata_svc/replication_spec_service.go:1327 # 0x1005adfec github.com/couchbase/goxdcr/metadata_svc.(*ReplicationSpecService).callMetadataChangeCb+0x2ac /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/metadata_svc/replication_spec_service.go:1298 # 0x1005ad2ba github.com/couchbase/goxdcr/metadata_svc.(*ReplicationSpecService).updateCacheInternal+0x3ba /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/metadata_svc/replication_spec_service.go:1195 # 0x1005acd34 github.com/couchbase/goxdcr/metadata_svc.(*ReplicationSpecService).updateCache+0x214 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/metadata_svc/replication_spec_service.go:1159 # 0x1005acd35 github.com/couchbase/goxdcr/metadata_svc.(*ReplicationSpecService).ReplicationSpecServiceCallback+0x215 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/metadata_svc/replication_spec_service.go:1122 # 0x100a438be github.com/couchbase/goxdcr/replication_manager.(*MetakvChangeListener).metakvCallback_async+0x5e /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/replication_manager/metakv_change_listener.go:97 1 @ 0x10003cab6 0x10004d493 0x10004d46d 0x100068985 0x100083905 0x1005acf8e 0x1005acf6a 0x1005aa76a 0x1005aa5fe 0x100a58804 0x100a3ec0d 0x100a3a9c8 0x100a3a1e5 0x1005b5b51 0x10006cfe1 # 0x100068984 sync.runtime_SemacquireMutex+0x24 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/runtime/sema.go:71 # 0x100083904 sync.(*Mutex).lockSlow+0x164 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/sync/mutex.go:162 # 0x1005acf8d sync.(*Mutex).Lock+0x8d /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/sync/mutex.go:81 # 0x1005acf69 github.com/couchbase/goxdcr/metadata_svc.(*ReplicationSpecService).updateCacheInternal+0x69 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/metadata_svc/replication_spec_service.go:1164 # 0x1005aa769 github.com/couchbase/goxdcr/metadata_svc.(*ReplicationSpecService).setReplicationSpecInternal+0x129 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/metadata_svc/replication_spec_service.go:818 # 0x1005aa5fd github.com/couchbase/goxdcr/metadata_svc.(*ReplicationSpecService).SetReplicationSpec+0x1d /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/metadata_svc/replication_spec_service.go:798 # 0x100a58803 github.com/couchbase/goxdcr/replication_manager.UpdateReplicationSettings+0x803 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/replication_manager/replication_manager.go:725 # 0x100a3ec0c github.com/couchbase/goxdcr/replication_manager.(*Adminport).doChangeReplicationSettingsRequest+0x5cc /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/replication_manager/adminport.go:737 # 0x100a3a9c7 github.com/couchbase/goxdcr/replication_manager.(*Adminport).handleRequest+0x727 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/replication_manager/adminport.go:217 # 0x100a3a1e4 github.com/couchbase/goxdcr/replication_manager.(*Adminport).processRequest+0x64 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/replication_manager/adminport.go:160 # 0x1005b5b50 github.com/couchbase/goxdcr/gen_server.(*GenServer).run+0x350 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/gen_server/gen_server.go:103 1 @ 0x100066d65 0x1003ae9b5 0x1003ae7cd 0x1003ab70b 0x1003ba3e5 0x1003baf9e 0x1002e91cf 0x1002eab69 0x1002ec7bb 0x1002e7c77 0x10006cfe1 # 0x100066d64 runtime/pprof.runtime_goroutineProfileWithLabels+0x24 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/runtime/mprof.go:753 # 0x1003ae9b4 runtime/pprof.writeRuntimeProfile+0xb4 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/runtime/pprof/pprof.go:725 # 0x1003ae7cc runtime/pprof.writeGoroutine+0x4c /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/runtime/pprof/pprof.go:685 # 0x1003ab70a runtime/pprof.(*Profile).WriteTo+0x14a /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/runtime/pprof/pprof.go:332 # 0x1003ba3e4 net/http/pprof.handler.ServeHTTP+0x4a4 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/pprof/pprof.go:253 # 0x1003baf9d net/http/pprof.Index+0x13d /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/pprof/pprof.go:371 # 0x1002e91ce net/http.HandlerFunc.ServeHTTP+0x2e /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/server.go:2084 # 0x1002eab68 net/http.(*ServeMux).ServeHTTP+0x148 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/server.go:2462 # 0x1002ec7ba net/http.serverHandler.ServeHTTP+0x43a /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/server.go:2916 # 0x1002e7c76 net/http.(*conn).serve+0x5d6 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/net/http/server.go:1966 1 @ 0x10006969f 0x1000b1149 0x1000cac05 0x1000cabed 0x1000ca9a5 0x1000d317e 0x1000d3176 0x100153dc3 0x10015458c 0x100a5b02a 0x10006cfe1 # 0x10006969e syscall.syscall+0x7e /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/runtime/sys_darwin.go:22 # 0x1000b1148 syscall.read+0x48 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/syscall/zsyscall_darwin_amd64.go:1189 # 0x1000cac04 syscall.Read+0x284 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/syscall/syscall_unix.go:188 # 0x1000cabec internal/poll.ignoringEINTRIO+0x26c /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_unix.go:794 # 0x1000ca9a4 internal/poll.(*FD).Read+0x24 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/internal/poll/fd_unix.go:163 # 0x1000d317d os.(*File).read+0x5d /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/os/file_posix.go:31 # 0x1000d3175 os.(*File).Read+0x55 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/os/file.go:119 # 0x100153dc2 bufio.(*Reader).fill+0x102 /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/bufio/bufio.go:106 # 0x10015458b bufio.(*Reader).ReadByte+0x2b /Users/neil.huang/.cbdepscache/exploded/x86_64/go-1.18.7/go/src/bufio/bufio.go:264 # 0x100a5b029 github.com/couchbase/goxdcr/replication_manager.pollStdin+0x149 /Users/neil.huang/source/couchbase/goproj/src/github.com/couchbase/goxdcr/replication_manager/replication_manager.go:1030