Ingestion hung, bad certificate, ingestion job JID:0.2 failed, Recovery stated but failed to perform storage cleanup

2024-06-05T20:53:00.058+00:00 ERRO CBAS.rebalance.TopologyMonitor [Topology Monitor] failed to perform storage cleanup; keep partitions [0, -1, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27, 32, 33, 34, 35, 40, 41, 42, 43, 48, 49, 50, 51, 56, 57, 58, 59, 64, 65, 66, 67, 72, 73, 74, 75, 80, 81, 82, 83, 88, 89, 90, 91, 96, 97, 98, 99, 104, 105, 106, 107, 112, 113, 114, 115, 120, 121, 122, 123]

org.apache.hyracks.api.exceptions.HyracksDataException: java.util.concurrent.ExecutionException: software.amazon.awssdk.services.s3.model.S3Exception: null (Service: S3, Status Code: 404, Request ID: 1YF21PTDP0BEXZ13, Extended Request ID: r0KQBrn1H0Tu7T9/Gyx5WkihTPDyezqKcXk20TB5EUB9G2QIc2D7g5u6nToxdqu81n86G5Rymrg=)

        at org.apache.hyracks.api.exceptions.HyracksDataException.create(HyracksDataException.java:49) ~[hyracks-api.jar:1.0.0-2126]

        at org.apache.asterix.cloud.clients.aws.s3.S3ParallelDownloader.downloadFiles(S3ParallelDownloader.java:76) ~[asterix-cloud.jar:1.0.0-2126]

        at org.apache.asterix.cloud.LazyCloudIOManager.downloadMetadataFiles(LazyCloudIOManager.java:242) ~[asterix-cloud.jar:1.0.0-2126]

        at org.apache.asterix.cloud.LazyCloudIOManager.downloadPartitions(LazyCloudIOManager.java:131) ~[asterix-cloud.jar:1.0.0-2126]

        at org.apache.asterix.cloud.AbstractCloudIOManager.bootstrap(AbstractCloudIOManager.java:133) ~[asterix-cloud.jar:1.0.0-2126]

        at com.couchbase.analytics.control.rebalance.TopologyMonitor$TopologyMonitorThread.keepPartitions(TopologyMonitor.java:207) ~[columnar-server.jar:1.0.0-2126]

        at com.couchbase.analytics.control.rebalance.TopologyMonitor$TopologyMonitorThread.cleanupStorage(TopologyMonitor.java:194) [columnar-server.jar:1.0.0-2126]

        at com.couchbase.analytics.control.rebalance.TopologyMonitor$TopologyMonitorThread.ensureTopology(TopologyMonitor.java:153) [columnar-server.jar:1.0.0-2126]

        at com.couchbase.analytics.control.rebalance.TopologyMonitor$TopologyMonitorThread.run(TopologyMonitor.java:114) [columnar-server.jar:1.0.0-2126]

        at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) [?:?]

        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]

        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]

        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]

        at java.base/java.lang.Thread.run(Thread.java:840) [?:?]

Caused by: java.util.concurrent.ExecutionException: software.amazon.awssdk.services.s3.model.S3Exception: null (Service: S3, Status Code: 404, Request ID: 1YF21PTDP0BEXZ13, Extended Request ID: r0KQBrn1H0Tu7T9/Gyx5WkihTPDyezqKcXk20TB5EUB9G2QIc2D7g5u6nToxdqu81n86G5Rymrg=)

        at java.base/java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]

        at java.base/java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]

        at org.apache.asterix.cloud.clients.aws.s3.S3ParallelDownloader.waitForFileDownloads(S3ParallelDownloader.java:130) ~[asterix-cloud.jar:1.0.0-2126]

        at org.apache.asterix.cloud.clients.aws.s3.S3ParallelDownloader.downloadFiles(S3ParallelDownloader.java:74) ~[asterix-cloud.jar:1.0.0-2126]

        ... 12 more

Caused by: software.amazon.awssdk.services.s3.model.S3Exception: null (Service: S3, Status Code: 404, Request ID: 1YF21PTDP0BEXZ13, Extended Request ID: r0KQBrn1H0Tu7T9/Gyx5WkihTPDyezqKcXk20TB5EUB9G2QIc2D7g5u6nToxdqu81n86G5Rymrg=)

        at software.amazon.awssdk.services.s3.model.S3Exception$BuilderImpl.build(S3Exception.java:104) ~[s3-2.24.9.jar:?]

        at software.amazon.awssdk.services.s3.model.S3Exception$BuilderImpl.build(S3Exception.java:58) ~[s3-2.24.9.jar:?]

        at software.amazon.awssdk.protocols.query.internal.unmarshall.AwsXmlErrorUnmarshaller.unmarshall(AwsXmlErrorUnmarshaller.java:99) ~[aws-query-protocol-2.24.9.jar:?]

        at software.amazon.awssdk.protocols.query.unmarshall.AwsXmlErrorProtocolUnmarshaller.handle(AwsXmlErrorProtocolUnmarshaller.java:102) ~[aws-query-protocol-2.24.9.jar:?]

        at software.amazon.awssdk.protocols.query.unmarshall.AwsXmlErrorProtocolUnmarshaller.handle(AwsXmlErrorProtocolUnmarshaller.java:82) ~[aws-query-protocol-2.24.9.jar:?]

        at software.amazon.awssdk.core.http.MetricCollectingHttpResponseHandler.lambda$handle$0(MetricCollectingHttpResponseHandler.java:52) ~[sdk-core-2.24.9.jar:?]

        at software.amazon.awssdk.core.internal.util.MetricUtils.measureDurationUnsafe(MetricUtils.java:99) ~[sdk-core-2.24.9.jar:?]

        at software.amazon.awssdk.core.internal.util.MetricUtils.measureDurationUnsafe(MetricUtils.java:92) ~[sdk-core-2.24.9.jar:?]

        at software.amazon.awssdk.core.http.MetricCollectingHttpResponseHandler.handle(MetricCollectingHttpResponseHandler.java:52) ~[sdk-core-2.24.9.jar:?]

        at software.amazon.awssdk.core.internal.http.async.AsyncResponseHandler.lambda$prepare$0(AsyncResponseHandler.java:92) ~[sdk-core-2.24.9.jar:?]

        at java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1150) ~[?:?]

        at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]

        at java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]

        at software.amazon.awssdk.core.internal.http.async.AsyncResponseHandler$BaosSubscriber.onComplete(AsyncResponseHandler.java:135) ~[sdk-core-2.24.9.jar:?]

        at software.amazon.awssdk.core.internal.metrics.BytesReadTrackingPublisher$BytesReadTracker.onComplete(BytesReadTrackingPublisher.java:74) ~[sdk-core-2.24.9.jar:?]

        at software.amazon.awssdk.utils.async.SimplePublisher.doProcessQueue(SimplePublisher.java:275) ~[utils-2.24.9.jar:?]