Internal Error while dropping dataset with columnar on-prem setup.

2024-05-08T20:48:45.013+13:00 ERRO CBAS.apache.asterix [Executor-12:ClusterController] Unexpected exception

java.lang.IllegalStateException: System is inconsistent state: pending dataset(Default.wa1dAZ1IaGY) couldn't be removed from the metadata

        at org.apache.asterix.app.translator.QueryTranslator.doDropDataset(QueryTranslator.java:2438) ~[asterix-app.jar:1.0.0-2057]

        at com.couchbase.analytics.lang.CBStatementExecutor.doDropDataset(CBStatementExecutor.java:1562) ~[columnar-connector.jar:1.0.0-2057]

        at org.apache.asterix.app.translator.QueryTranslator.handleDatasetDropStatement(QueryTranslator.java:2343) ~[asterix-app.jar:1.0.0-2057]

        at com.couchbase.analytics.lang.CBStatementExecutor.handleDatasetDropStatement(CBStatementExecutor.java:1542) ~[columnar-connector.jar:1.0.0-2057]

        at org.apache.asterix.app.translator.QueryTranslator.compileAndExecute(QueryTranslator.java:415) ~[asterix-app.jar:1.0.0-2057]

        at org.apache.asterix.app.message.ExecuteStatementRequestMessage.handle(ExecuteStatementRequestMessage.java:181) ~[asterix-app.jar:1.0.0-2057]

        at org.apache.asterix.messaging.CCMessageBroker.receivedMessage(CCMessageBroker.java:64) ~[asterix-app.jar:1.0.0-2057]

        at org.apache.hyracks.control.cc.work.ApplicationMessageWork.lambda$notifyMessageBroker$0(ApplicationMessageWork.java:74) ~[hyracks-control-cc.jar:1.0.0-2057]

        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]

        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]

        at java.base/java.lang.Thread.run(Thread.java:840) [?:?]

Caused by: java.lang.InterruptedException

        at java.base/java.lang.Object.wait(Native Method) ~[?:?]

        at java.base/java.lang.Object.wait(Object.java:338) ~[?:?]

        at org.apache.hyracks.control.cc.job.JobRun.waitForCompletion(JobRun.java:236) ~[hyracks-control-cc.jar:1.0.0-2057]

        at org.apache.hyracks.control.cc.work.WaitForJobCompletionWork$1.run(WaitForJobCompletionWork.java:58) ~[hyracks-control-cc.jar:1.0.0-2057]

        ... 3 more

        Suppressed: org.apache.asterix.common.exceptions.AsterixException: ASX1050: Cannot find dataset with name wa1dAZ1IaGY in dataverse Default.Default

                at org.apache.asterix.metadata.MetadataNode.dropDataset(MetadataNode.java:993) ~[asterix-metadata.jar:1.0.0-2057]

                at org.apache.asterix.metadata.MetadataManager.dropDataset(MetadataManager.java:362) ~[asterix-metadata.jar:1.0.0-2057]

                at org.apache.asterix.app.translator.QueryTranslator.doDropDataset(QueryTranslator.java:2432) ~[asterix-app.jar:1.0.0-2057]

                at com.couchbase.analytics.lang.CBStatementExecutor.doDropDataset(CBStatementExecutor.java:1562) ~[columnar-connector.jar:1.0.0-2057]

                at org.apache.asterix.app.translator.QueryTranslator.handleDatasetDropStatement(QueryTranslator.java:2343) ~[asterix-app.jar:1.0.0-2057]

                at com.couchbase.analytics.lang.CBStatementExecutor.handleDatasetDropStatement(CBStatementExecutor.java:1542) ~[columnar-connector.jar:1.0.0-2057]

                at org.apache.asterix.app.translator.QueryTranslator.compileAndExecute(QueryTranslator.java:415) ~[asterix-app.jar:1.0.0-2057]

                at org.apache.asterix.app.message.ExecuteStatementRequestMessage.handle(ExecuteStatementRequestMessage.java:181) ~[asterix-app.jar:1.0.0-2057]

                at org.apache.asterix.messaging.CCMessageBroker.receivedMessage(CCMessageBroker.java:64) ~[asterix-app.jar:1.0.0-2057]

                at org.apache.hyracks.control.cc.work.ApplicationMessageWork.lambda$notifyMessageBroker$0(ApplicationMessageWork.java:74) ~[hyracks-control-cc.jar:1.0.0-2057]

                at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]

                at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]

                at java.base/java.lang.Thread.run(Thread.java:840) [?:?]