[System Test][XDCR] error observed in longevity - MCResponse status=KEY_ENOENT, opcode=0x89, opaque=0, msg:

2021-11-09T20:05:35.638-08:00 INFO GOXDCR.RuntimeCtx: d0851ce9e1015bf9cebb3f08822e56de/bucket4/bucket4 successfully stopped service BackfillMgrSvc.

2021-11-09T20:05:35.638-08:00 INFO GOXDCR.StatsMgr: d0851ce9e1015bf9cebb3f08822e56de/bucket4/bucket4-567737452 expvar=Stats for pipeline d0851ce9e1015bf9cebb3f08822e56de/bucket4/bucket4-567737452 {"Backfill Old Progress": "Source nozzles have been closed", "Backfill Progress": "Pipeline has been stopped", "Errors": "[{\"time\":\"2021-11-09T20:05:25.030536071-08:00\",\"errMsg\":\"dcp_d0851ce9e1015bf9cebb3f08822e56de/bucket4/bucket4_172.23.99.21:11210_0:MCResponse status=KEY_ENOENT, opcode=0x89, opaque=0, msg: \"},{\"time\":\"2021-11-09T20:05:07.940021281-08:00\",\"errMsg\":\"dcp_d0851ce9e1015bf9cebb3f08822e56de/bucket4/bucket4_172.23.104.5:11210_0:MCResponse status=KEY_ENOENT, opcode=0x89, opaque=0, msg: \"},{\"time\":\"2021-11-09T20:04:51.148858282-08:00\",\"errMsg\":\"dcp_d0851ce9e1015bf9cebb3f08822e56de/bucket4/bucket4_172.23.99.21:11210_0:MCResponse status=KEY_ENOENT, opcode=0x89, opaque=0, msg: \"},{\"time\":\"2021-11-09T20:04:34.30676171-08:00\",\"errMsg\":\"dcp_d0851ce9e1015bf9cebb3f08822e56de/bucket4/bucket4_172.23.104.5:11210_0:MCResponse status=KEY_ENOENT, opcode=0x89, opaque=0, msg: \"},{\"time\":\"2021-11-09T20:04:17.446469141-08:00\",\"errMsg\":\"dcp_d0851ce9e1015bf9cebb3f08822e56de/bucket4/bucket4_172.23.104.70:11210_1:MCResponse status=KEY_ENOENT, opcode=0x89, opaque=0, msg: \"},{\"time\":\"2021-11-09T20:04:00.570547128-08:00\",\"errMsg\":\"dcp_d0851ce9e1015bf9cebb3f08822e56de/bucket4/bucket4_172.23.97.121:11210_0:MCResponse status=KEY_ENOENT, opcode=0x89, opaque=0, msg: \"},{\"time\":\"2021-11-09T20:03:43.402213497-08:00\",\"errMsg\":\"dcp_d0851ce9e1015bf9cebb3f08822e56de/bucket4/bucket4_172.23.97.122:11210_0:MCResponse status=KEY_ENOENT, opcode=0x89, opaque=0, msg: \"},{\"time\":\"2021-11-09T20:03:26.480499628-08:00\",\"errMsg\":\"dcp_d0851ce9e1015bf9cebb3f08822e56de/bucket4/bucket4_172.23.108.103:11210_1:MCResponse status=KEY_ENOENT, opcode=0x89, opaque=0, msg: \"},{\"time\":\"2021-11-09T20:03:09.60922229-08:00\",\"errMsg\":\"dcp_d0851ce9e1015bf9cebb3f08822e56de/bucket4/bucket4_172.23.99.20:11210_0:MCResponse status=KEY_ENOENT, opcode=0x89, opaque=0, msg: \"},{\"time\":\"2021-11-09T20:02:52.746939256-08:00\",\"errMsg\":\"dcp_d0851ce9e1015bf9cebb3f08822e56de/bucket4/bucket4_172.23.99.21:11210_0:MCResponse status=KEY_ENOENT, opcode=0x89, opaque=0, msg: \"},{\"time\":\"2021-11-09T20:02:35.812363461-08:00\",\"errMsg\":\"dcp_d0851ce9e1015bf9cebb3f08822e56de/bucket4/bucket4_172.23.99.20:11210_0:MCResponse status=KEY_ENOENT, opcode=0x89, opaque=0, msg: \"},{\"time\":\"2021-11-09T20:02:18.926237945-08:00\",\"errMsg\":\"dcp_d0851ce9e1015bf9cebb3f08822e56de/bucket4/bucket4_172.23.97.119:11210_0:MCResponse status=KEY_ENOENT, opcode=0x89, opaque=0, msg: \"},{\"time\":\"2021-11-09T20:02:02.165256985-08:00\",\"errMsg\":\"dcp_d0851ce9e1015bf9cebb3f08822e56de/bucket4/bucket4_172.23.104.5:11210_1:MCResponse status=KEY_ENOENT, opcode=0x89, opaque=0, msg: \"},{\"time\":\"2021-11-09T20:01:45.304130926-08:00\",\"errMsg\":\"dcp_d0851ce9e1015bf9cebb3f08822e56de/bucket4/bucket4_172.23.104.5:11210_0:MCResponse status=KEY_ENOENT, opcode=0x89, opaque=0, msg: \"},{\"time\":\"2021-11-09T20:01:28.339666586-08:00\",\"errMsg\":\"dcp_d0851ce9e1015bf9cebb3f08822e56de/bucket4/bucket4_172.23.99.21:11210_1:MCResponse status=KEY_ENOENT, opcode=0x89, opaque=0, msg: \"},{\"time\":\"2021-11-09T20:01:11.923660761-08:00\",\"errMsg\":\"pipelineMgr.pipeline_factory.NewPipeline(d0851ce9e1015bf9cebb3f08822e56de/bucket4/bucket4):Invalid configuration. No source nozzle can be constructed since the source kv nodes are not the master for any vbuckets.\"},{\"time\":\"2021-11-09T20:01:01.005353943-08:00\",\"errMsg\":\"dcp_d0851ce9e1015bf9cebb3f08822e56de/bucket4/bucket4_172.23.104.5:11210_0:MCResponse status=KEY_ENOENT, opcode=0x89, opaque=0, msg: \"},{\"time\":\"2021-11-09T20:00:43.610128097-08:00\",\"errMsg\":\"dcp_d0851ce9e1015bf9cebb3f08822e56de/bucket4/bucket4_172.23.104.5:11210_0:MCResponse status=KEY_ENOENT, opcode=0x89, opaque=0, msg: \"},{\"time\":\"2021-11-09T20:00:26.397306396-08:00\",\"errMsg\":\"dcp_d0851ce9e1015bf9cebb3f08822e56de/bucket4/bucket4_172.23.97.119:11210_0:MCResponse status=KEY_ENOENT, opcode=0x89, opaque=0, msg: \"},{\"time\":\"2021-11-09T20:00:09.196211077-08:00\",\"errMsg\":\"dcp_d0851ce9e1015bf9cebb3f08822e56de/bucket4/bucket4_172.23.108.103:11210_1:MCResponse status=KEY_ENOENT, opcode=0x89, opaque=0, msg: \"}]", "OldProgress": "Pipeline failed to start, err=map[dcp_d0851ce9e1015bf9cebb3f08822e56de/bucket4/bucket4_172.23.99.21:11210_0:MCResponse status=KEY_ENOENT, opcode=0x89, opaque=0, msg: ]", "Overview": {"CurrentTime": 1636517135638198456, "add_docs_written": 0, "bandwidth_usage": 0, "changes_left": 0, "data_merged": 0, "data_replicated": 0, "datapool_failed_gets": 0, "dcp_datach_length": 0, "dcp_dispatch_time": 0, "deletion_docs_written": 0, "deletion_failed_cr_source": 0, "deletion_filtered": 0, "deletion_received_from_dcp": 0, "docs_checked": 0, "docs_cloned": 0, "docs_failed_cr_source": 0, "docs_filtered": 0, "docs_merged": 0, "docs_opt_repd": 0, "docs_processed": 0, "docs_received_from_dcp": 0, "docs_rep_queue": 0, "docs_unable_to_filter": 0, "docs_written": 0, "expiry_docs_merged": 0, "expiry_docs_written": 0, "expiry_failed_cr_source": 0, "expiry_filtered": 0, "expiry_received_from_dcp": 0, "expiry_stripped": 0, "num_checkpoints": 0, "num_failedckpts": 0, "rate_doc_checks": 0, "rate_doc_opt_repd": 0, "rate_received_from_dcp": 0, "rate_replicated": 0, "resp_wait_time": 0, "set_docs_written": 0, "set_failed_cr_source": 0, "set_filtered": 0, "set_received_from_dcp": 0, "size_rep_queue": 0, "target_docs_skipped": 0, "throttle_latency": 0, "throughput_throttle_latency": 0, "time_committing": 0, "wtavg_docs_latency": 0, "wtavg_get_doc_latency": 0, "wtavg_merge_latency": 0, "wtavg_meta_latency": 0}, "Progress": "Async listeners have been stopped", "Status": "Pending"}