[System Test][CBBS] three tasks failed with error - Cannot dispatch task as the task is already marked as running and failed the check {"err": "task is already running or waiting to run"}

{

  "task_name": "backup-1",

  "status": "failed",

  "start": "2022-07-10T18:00:32.825095129-07:00",

  "end": "2022-07-10T18:00:32.853720789-07:00",

  "error": "task is already running or waiting to run",

  "error_code": 5,

  "type": "BACKUP",

  "show": true

}

2022-07-09T00:00:16.866-07:00 DEBUG (ClockKeeper) Job triggered {"job": {"TaskName":"backup-1","ClusterName":"self","RepositoryName":"my_repo","NextScheduledRun":"2022-07-09T06:00:16-07:00"}}

2022-07-09T00:00:16.866-07:00 INFO (Dispatcher) Dispatching task {"clusterName": "self", "repositoryID": "my_repo", "task": "backup-1"}

2022-07-09T00:00:16.972-07:00 WARN (Dispatcher) Task seems to already be running, checking if it is orphan {"runningTask": {"task_name":"backup-1","status":"running","start":"2022-07-08T18:00:15.117441485-07:00","node_runs":[{"node_id":"a58f4e6c708c388c4479ef532ce4a82f","status":"running","start":"2022-07-08T21:59:11.902181959-07:00","end":"0001-01-01T00:00:00Z","progress":76.68343653250774,"stats":{"id":"3be861c1-0ff7-421e-8bb2-ea90746cc311","current_transfer":1,"total_transfers":1,"transfers":[{"description":"Backing up to 2022-07-08T21_59_13.229814764-07_00","stats":{"started_at":1657342752332951005,"buckets":{"ITEM":{"estimated_total_items":308545072,"total_items":22887,"total_vbuckets":1024,"vbuckets_complete":682,"bytes_received":30419083,"snapshot_markers_received":722,"failover_logs_received":1024,"mutations_received":16670,"deletions_received":264395,"started_at":1657349990453689347},"NEW_ORDER":{"estimated_total_items":188569214,"total_items":1890319,"total_vbuckets":1024,"vbuckets_complete":1024,"bytes_received":329092648,"snapshot_markers_received":1024,"failover_logs_received":1024,"mutations_received":1897156,"deletions_received":2539112,"started_at":1657342758465663371,"finished_at":1657343018777666108,"complete":true},"WAREHOUSE":{"estimated_total_items":144019545,"total_items":2359650,"total_vbuckets":1024,"vbuckets_complete":1024,"bytes_received":2856358464,"snapshot_markers_received":1024,"failover_logs_received":1024,"mutations_received":2304529,"deletions_received":2233411,"started_at":1657343022450380438,"finished_at":1657349622457517713,"complete":true},"bucket4":{"estimated_total_items":72511949,"total_vbuckets":1024,"vbuckets_complete":1024,"bytes_received":134692,"snapshot_markers_received":1024,"failover_logs_received":1024,"deletions_received":956,"started_at":1657349625954560481,"finished_at":1657349629114618147,"complete":true},"bucket5":{"estimated_total_items":44801024,"total_vbuckets":1024,"vbuckets_complete":1024,"bytes_received":3019978,"snapshot_markers_received":1024,"failover_logs_received":1024,"deletions_received":57846,"started_at":1657349632810119751,"finished_at":1657349635649250219,"complete":true},"bucket6":{"estimated_total_items":152225951,"total_vbuckets":1024,"vbuckets_complete":1024,"bytes_received":22771464,"snapshot_markers_received":1024,"failover_logs_received":1024,"deletions_received":447484,"started_at":1657349639509625434,"finished_at":1657349646836747760,"complete":true},"bucket9":{"estimated_total_items":1196104,"total_vbuckets":1024,"vbuckets_complete":1024,"bytes_received":86528,"snapshot_markers_received":1024,"failover_logs_received":1024,"started_at":1657349650439338316,"finished_at":1657349656475360263,"complete":true},"default":{"estimated_total_items":591066628,"total_items":912753,"total_vbuckets":1024,"vbuckets_complete":1024,"bytes_received":7622449243,"snapshot_markers_received":1024,"failover_logs_received":1024,"mutations_received":880226,"deletions_received":3595680,"started_at":1657349659896287827,"finished_at":1657349986701199357,"complete":true}}},"progress":76.68343653250774,"eta":"2022-07-09T00:36:57.253853721-07:00"}],"progress":76.68343653250774,"eta":"2022-07-09T00:36:57.253853721-07:00"},"error_code":0}],"error_code":0,"type":"BACKUP"}}

2022-07-09T00:00:16.997-07:00 INFO (GRPC Manager) Orphan check {"cluster": "self", "repo": "my_repo", "state": "ACTIVE", "task": "backup-1", "returnStatus": "running"}

2022-07-09T00:00:16.998-07:00 ERROR (Dispatcher) Cannot dispatch task as the task is already marked as running and failed the check {"err": "task is already running or waiting to run"}