[BP 6.6.3] Internal error while dropping connected remote datasets

2021-06-17T04:53:55.364-07:00 ERRO CBAS.context.DatasetLifecycleManager [SAO:JID:0.10:TAID:TID:ANID:ODID:0:0:0:0] Failed to drop in-use index storage/partition_0/x/y/0/y. Ref count (2), Operation tracker active ops (0)

2021-06-17T04:53:55.366-07:00 WARN CBAS.nc.Task [SA:JID:0.10:TAID:TID:ANID:ODID:0:0:0:0:0] Task failed with exception

org.apache.hyracks.api.exceptions.HyracksDataException: HYR0105: Cannot drop in-use index (y)

        at org.apache.hyracks.api.exceptions.HyracksDataException.create(HyracksDataException.java:57) ~[hyracks-api.jar:7.0.0-5295]

        at org.apache.asterix.common.context.DatasetLifecycleManager.unregister(DatasetLifecycleManager.java:163) ~[asterix-common.jar:7.0.0-5295]

        at org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelper.destroy(IndexDataflowHelper.java:97) ~[hyracks-storage-am-common.jar:7.0.0-5295]

        at org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorNodePushable.dropIndex(IndexDropOperatorNodePushable.java:82) ~[hyracks-storage-am-common.jar:7.0.0-5295]

        at org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorNodePushable.initialize(IndexDropOperatorNodePushable.java:71) ~[hyracks-storage-am-common.jar:7.0.0-5295]

        at org.apache.hyracks.api.rewriter.runtime.SuperActivityOperatorNodePushable.lambda$runInParallel$0(SuperActivityOperatorNodePushable.java:227) ~[hyracks-api.jar:7.0.0-5295]

        at java.util.concurrent.FutureTask.run(Unknown Source) ~[?:?]

        at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) [?:?]

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) [?:?]

        at java.lang.Thread.run(Unknown Source) [?:?]

2021-06-17T04:53:55.377-07:00 WARN CBAS.work.NotifyTaskFailureWork [Worker:2a3668cdd8829b616659d77d461694bd] task TAID:TID:ANID:ODID:0:0:0:0 has failed

org.apache.hyracks.api.exceptions.HyracksDataException: HYR0105: Cannot drop in-use index (y)

        at org.apache.hyracks.api.exceptions.HyracksDataException.create(HyracksDataException.java:70) ~[hyracks-api.jar:7.0.0-5295]

        at org.apache.hyracks.api.util.ExceptionUtils.setNodeIds(ExceptionUtils.java:70) ~[hyracks-api.jar:7.0.0-5295]

        at org.apache.hyracks.control.nc.Task.run(Task.java:390) ~[hyracks-control-nc.jar:7.0.0-5295]

        at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) ~[?:?]

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) ~[?:?]

        at java.lang.Thread.run(Unknown Source) ~[?:?]

2021-06-17T04:53:55.384-07:00 WARN CBAS.work.TaskFailureWork [Worker:ClusterController] Executing task failure work for TaskFailure: [JID:0.10:TAID:TID:ANID:ODID:0:0:0:0:2a3668cdd8829b616659d77d461694bd]

org.apache.hyracks.api.exceptions.HyracksDataException: HYR0105: Cannot drop in-use index (y)

        at org.apache.hyracks.api.exceptions.HyracksDataException.create(HyracksDataException.java:70) ~[hyracks-api.jar:7.0.0-5295]

        at org.apache.hyracks.api.util.ExceptionUtils.setNodeIds(ExceptionUtils.java:70) ~[hyracks-api.jar:7.0.0-5295]

        at org.apache.hyracks.control.nc.Task.run(Task.java:390) ~[hyracks-control-nc.jar:7.0.0-5295]

        at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) ~[?:?]

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) ~[?:?]

        at java.lang.Thread.run(Unknown Source) ~[?:?]

2021-06-17T04:53:55.386-07:00 INFO CBAS.work.JobCleanupWork [Worker:ClusterController] Cleanup for job: JID:0.10

2021-06-17T04:53:55.397-07:00 ERRO CBAS.translator.QueryTranslator [QueryTranslator:5c26330e-b6ac-448b-b4c5-50e7410e8f9b] failed to drop dataset; executing compensating operations

org.apache.hyracks.api.exceptions.HyracksDataException: HYR0105: Cannot drop in-use index (y)

        at org.apache.hyracks.api.exceptions.HyracksDataException.create(HyracksDataException.java:70) ~[hyracks-api.jar:7.0.0-5295]

        at org.apache.hyracks.api.util.ExceptionUtils.setNodeIds(ExceptionUtils.java:70) ~[hyracks-api.jar:7.0.0-5295]

        at org.apache.hyracks.control.nc.Task.run(Task.java:390) ~[hyracks-control-nc.jar:7.0.0-5295]

        at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) [?:?]

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) [?:?]

        at java.lang.Thread.run(Unknown Source) [?:?]

2021-06-17T04:53:55.399-07:00 INFO CBAS.utils.DatasetUtil [QueryTranslator:5c26330e-b6ac-448b-b4c5-50e7410e8f9b] DROP DATASET: {"bucketName":"beer-sample","whereClause":null,"nodeGroupName":"x.y","hints":{},"bucketDataverseName":{"canonicalForm":"Default","parts":["Default"],"partCount":1},"scopeName":"_default","datasetName":"y","recordTypeDataverseName":{"canonicalForm":"Metadata","parts":["Metadata"],"partCount":1},"datasetDetails":"org.apache.asterix.metadata.entities.InternalDatasetDetails@47c5a5ca","pendingOp":"Pending No Operation","collectionName":"_default","metaTypeName":"DCPMeta","rebalanceCount":0,"datasetId":"101","compactionPolicyFactory":"concurrent","compactionPolicyProperties":{"max-component-count":"30","min-merge-component-count":"3","max-merge-component-count":"10","size-ratio":"1.2"},"metaTypeDataverseName":{"canonicalForm":"Metadata","parts":["Metadata"],"partCount":1},"dataverseName":{"canonicalForm":"x","parts":["x"],"partCount":1},"bucketLinkName":"link1","datasetType":"INTERNAL","collectionId":"0x0","compressionScheme":"snappy","recordTypeName":"AnyObject"}

2021-06-17T04:53:55.406-07:00 WARN CBAS.apache.asterix [Executor-5:ClusterController] HYR0105: Cannot drop in-use index (y)

org.apache.hyracks.api.exceptions.HyracksDataException: HYR0105: Cannot drop in-use index (y)

        at org.apache.hyracks.api.exceptions.HyracksDataException.create(HyracksDataException.java:70) ~[hyracks-api.jar:7.0.0-5295]

        at org.apache.hyracks.api.util.ExceptionUtils.setNodeIds(ExceptionUtils.java:70) ~[hyracks-api.jar:7.0.0-5295]

        at org.apache.hyracks.control.nc.Task.run(Task.java:390) ~[hyracks-control-nc.jar:7.0.0-5295]

        at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) [?:?]

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) [?:?]

        at java.lang.Thread.run(Unknown Source) [?:?]

        Suppressed: org.apache.asterix.common.exceptions.ACIDException: Transaction TxnId:36 doesn't exist.

                at org.apache.asterix.transaction.management.service.transaction.TransactionManager.getTransactionContext(TransactionManager.java:74) ~[asterix-transactions.jar:7.0.0-5295]

                at org.apache.asterix.metadata.MetadataNode.modifyMetadataIndex(MetadataNode.java:470) ~[asterix-metadata.jar:7.0.0-5295]

                at org.apache.asterix.metadata.MetadataNode.deleteTupleFromIndex(MetadataNode.java:768) ~[asterix-metadata.jar:7.0.0-5295]

                at org.apache.asterix.metadata.MetadataNode.dropDataset(MetadataNode.java:659) ~[asterix-metadata.jar:7.0.0-5295]

                at org.apache.asterix.metadata.MetadataManager.dropDataset(MetadataManager.java:271) ~[asterix-metadata.jar:7.0.0-5295]

                at org.apache.asterix.metadata.entities.Dataset.drop(Dataset.java:367) ~[asterix-metadata.jar:7.0.0-5295]

                at com.couchbase.analytics.metadata.ShadowDataset.drop(ShadowDataset.java:173) ~[cbas-connector.jar:7.0.0-5295]

                at org.apache.asterix.app.translator.QueryTranslator.doDropDataset(QueryTranslator.java:1722) ~[asterix-app.jar:7.0.0-5295]

                at com.couchbase.analytics.lang.CBStatementExecutor.doDropDataset(CBStatementExecutor.java:782) ~[cbas-connector.jar:7.0.0-5295]

                at org.apache.asterix.app.translator.QueryTranslator.handleDatasetDropStatement(QueryTranslator.java:1661) ~[asterix-app.jar:7.0.0-5295]

                at com.couchbase.analytics.lang.CBStatementExecutor.handleDatasetDropStatement(CBStatementExecutor.java:768) ~[cbas-connector.jar:7.0.0-5295]

                at org.apache.asterix.app.translator.QueryTranslator.compileAndExecute(QueryTranslator.java:359) ~[asterix-app.jar:7.0.0-5295]

                at org.apache.asterix.app.message.ExecuteStatementRequestMessage.handle(ExecuteStatementRequestMessage.java:170) ~[asterix-app.jar:7.0.0-5295]

                at org.apache.asterix.messaging.CCMessageBroker.receivedMessage(CCMessageBroker.java:64) ~[asterix-app.jar:7.0.0-5295]

                at org.apache.hyracks.control.cc.work.ApplicationMessageWork.lambda$notifyMessageBroker$0(ApplicationMessageWork.java:68) ~[hyracks-control-cc.jar:7.0.0-5295]

                at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) [?:?]

                at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) [?:?]

                at java.lang.Thread.run(Unknown Source) [?:?]

2021-06-17T04:53:55.411-07:00 INFO CBAS.messaging.NCMessageBroker [Worker:2a3668cdd8829b616659d77d461694bd] Received message: ExecuteStatementResponseMessage(id=31): 0 characters

2021-06-17T04:53:55.425-07:00 WARN CBAS.server.QueryServiceServlet [HttpExecutor(port:8095)-3] handleException: unexpected exception: <ud>{"host":"10.112.205.101:8091","path":"/query/service","statement":"drop dataset `x`.`y`","pretty":false,"mode":"immediate","clientContextID":null,"format":"CLEAN_JSON","timeout":9223372036854775807,"maxResultReads":1,"planFormat":"JSON","expressionTree":false,"rewrittenExpressionTree":false,"logicalPlan":false,"optimizedLogicalPlan":false,"job":false,"profile":"counts","signature":true,"multiStatement":false,"parseOnly":false,"readOnly":false,"maxWarnings":0,"scanConsistency":null,"scanWait":null}</ud>

org.apache.hyracks.api.exceptions.HyracksDataException: HYR0105: Cannot drop in-use index (y)