[System Test] Queries fail with Internal error - at com.couchbase.analytics.auth.AuthorizationProviderCloud.ensureAuthorized exception

2024-07-10T17:00:35.054+00:00 INFO CBAS.server.QueryServiceServlet [HttpExecutor(port:18095)-12] handleRequest: uuid=df8ca3da-6a5a-4470-8ef3-ac6046c6f4f2, clientContextID=null, {"host":"cb.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com:18095","path":"/query/service","statement":"<ud>select count(*) from Database18DLsVwSJZ.scope1qTEZuKno.remotedatasetrEFRtOhD</ud>","pretty":false,"mode":"immediate","clientContextID":null,"clientType":"ASTERIX","dataverse":null,"format":"CLEAN_JSON","timeout":9223372036854775807,"maxResultReads":1,"planFormat":"JSON","expressionTree":false,"rewrittenExpressionTree":false,"logicalPlan":false,"optimizedLogicalPlan":false,"job":false,"profile":"counts","signature":true,"multiStatement":true,"parseOnly":false,"readOnly":false,"maxWarnings":0,"sqlCompat":false,"source":null,"scanConsistency":null,"scanWait":null}

2024-07-10T17:00:35.068+00:00 INFO CBAS.messaging.NCMessageBroker [Worker:662807a2bbb1cb60b40003a074f0bcec] Received message: ExecuteStatementResponseMessage(id=281, uuid=df8ca3da-6a5a-4470-8ef3-ac6046c6f4f2, clientContextId=null): 0 characters

2024-07-10T17:00:35.069+00:00 WARN CBAS.server.QueryServiceServlet [HttpExecutor(port:18095)-12] handleException: unexpected exception: uuid=null, clientContextID=df8ca3da-6a5a-4470-8ef3-ac6046c6f4f2

com.couchbase.analytics.common.exceptions.AnalyticsHyracksException: CBAS0022: Internal error

	at com.couchbase.analytics.auth.AuthorizationProviderCloud.ensureAuthorized(AuthorizationProviderCloud.java:112) ~[columnar-server.jar:1.0.0-2203]

	at com.couchbase.analytics.request.AnalyticsReceptionist.ensureSchedulable(AnalyticsReceptionist.java:33) ~[columnar-common.jar:1.0.0-2203]

	at org.apache.asterix.app.translator.QueryTranslator.createAndRunJob(QueryTranslator.java:5423) ~[asterix-app.jar:1.0.0-2203]

	at org.apache.asterix.app.translator.QueryTranslator.deliverResult(QueryTranslator.java:5310) ~[asterix-app.jar:1.0.0-2203]

	at org.apache.asterix.app.translator.QueryTranslator.handleQuery(QueryTranslator.java:5280) ~[asterix-app.jar:1.0.0-2203]

	at org.apache.asterix.app.translator.QueryTranslator.compileAndExecute(QueryTranslator.java:533) ~[asterix-app.jar:1.0.0-2203]

	at org.apache.asterix.app.message.ExecuteStatementRequestMessage.handle(ExecuteStatementRequestMessage.java:181) ~[asterix-app.jar:1.0.0-2203]

	at org.apache.asterix.messaging.CCMessageBroker.receivedMessage(CCMessageBroker.java:64) ~[asterix-app.jar:1.0.0-2203]

	at org.apache.hyracks.control.cc.work.ApplicationMessageWork.lambda$notifyMessageBroker$0(ApplicationMessageWork.java:74) ~[hyracks-control-cc.jar:1.0.0-2203]

	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]

	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]

	at java.base/java.lang.Thread.run(Thread.java:840) [?:?]

2024-07-11T07:02:48.008+00:00 INFO CBAS.messaging.CCMessageBroker [Executor-976:ClusterController] Received message: ExecuteStatementRequestMessage(id=325, from=e71ff082812e12d4c853d428a9a7f9c2, uuid=9c843350-ca01-4a25-b67d-7001edda7d94, clientContextID=null): <ud>\nSELECT meta.*\nFROM (\n  SELECT ds.DatabaseName,\n  ds.DataverseName,\nds.DataverseName || '.' || ds.DatasetName AS datasetFullyQualifiedName,\n  ds.DatasetName AS id,\n  TRUE AS isDataset,\n  ds.BucketName AS bucketName,\n  ds.ScopeName AS scopeName,\n  ds.CollectionName AS collectionName,\n  ds.BucketDataverseName AS linkDataverseName,\n  ds.BucketDatabaseName AS linkDatabaseName,\n  ds.LinkDataverseName AS linkDataverseName2,\n  ds.LinkDatabaseName AS linkDatabaseName2,\n  ds.`Filter` AS `filter`,\n  ds.LinkName,\n  ds.DatasetType,\n  ds.DatatypeName,\n  ds.InternalDetails.PrimaryKey as primaryKey,\n  ds.InternalDetails.PrimaryKeyTypes as primaryKeyTypes,\n  ds.InternalDetails.`Autogenerated` as primaryKeyAutogenerated,\n  ds.ViewDetails,\n  t.Derived.Record.Fields AS ViewFields,\n  concat2(', ', (\n    SELECT VALUE FieldName || ' ' || ( CASE WHEN LOWER(FieldType) = 'int64' THEN 'BIGINT' ELSE UPPER(FieldType) END) || ( CASE WHEN NOT IsNullable\nAND NOT IsMissable THEN ' NOT UNKNOWN' ELSE '' END)\nFROM t.Derived.Record.Fields)) AS TypeString,\n  (\n    SELECT idx.IndexName,\n  idx.SearchKey,\n  idx.SearchKeyType\nFROM Metadata.`Index` AS idx\nWHERE idx.IsPrimary = FALSE\nAND idx.DatabaseName = ds.DatabaseName\nAND idx.DatasetName = ds.DatasetName\nAND idx.DataverseName = ds.DataverseName) AS indexes,\n  ds.ExternalDetails.Properties AS externalDetails\nFROM Metadata.`Dataset` AS ds LEFT\nJOIN Metadata.Datatype t ON ds.DatabaseName = t.DatabaseName AND ds.DataverseName = t.DataverseName\nAND t.DatatypeName = ds.DatatypeName\nWHERE (ds.DataverseName != 'Metadata'\n)\nUNION ALL\nSELECT dv.DatabaseName,\n  dv.DataverseName,\n  TRUE AS isDataverse,\n  (\n    SELECT l.Name\nFROM Metadata.`Link` AS l\nWHERE l.DatabaseName = dv.DatabaseName AND l.DataverseName = dv.DataverseName) AS links\nFROM Metadata.`Dataverse` AS dv\nWHERE dv.DataverseName != 'Metadata'\n\n  UNION ALL\nSELECT db.DatabaseName,\n  TRUE AS isDatabase\nFROM Metadata.`Database` AS db\nWHERE db.DatabaseName != 'System'\n\n  UNION ALL\nSELECT DatabaseName,\n  DataverseName,\n  Name,\n  IsActive,\n  ClusterName,\n  `Type` AS LinkType,\n  TRUE AS isLink\nFROM Metadata.`Link`) meta\nORDER BY\n  meta.isDatabase DESC,\n  meta.isDataverse DESC,\n  meta.isLink DESC,\n  meta.DatabaseName,\n  meta.DataverseName,\n  meta.LinkName,\n  meta.id;</ud>

2024-07-11T07:02:48.078+00:00 WARN CBAS.apache.asterix [Executor-976:ClusterController] CBAS0022: Internal error

com.couchbase.analytics.common.exceptions.AnalyticsHyracksException: CBAS0022: Internal error

	at com.couchbase.analytics.auth.AuthorizationProviderCloud.ensureAuthorized(AuthorizationProviderCloud.java:112) ~[columnar-server.jar:1.0.0-2203]

	at com.couchbase.analytics.request.AnalyticsReceptionist.ensureSchedulable(AnalyticsReceptionist.java:33) ~[columnar-common.jar:1.0.0-2203]

	at org.apache.asterix.app.translator.QueryTranslator.createAndRunJob(QueryTranslator.java:5423) ~[asterix-app.jar:1.0.0-2203]

	at org.apache.asterix.app.translator.QueryTranslator.deliverResult(QueryTranslator.java:5310) ~[asterix-app.jar:1.0.0-2203]

	at org.apache.asterix.app.translator.QueryTranslator.handleQuery(QueryTranslator.java:5280) ~[asterix-app.jar:1.0.0-2203]

	at org.apache.asterix.app.translator.QueryTranslator.compileAndExecute(QueryTranslator.java:533) ~[asterix-app.jar:1.0.0-2203]

	at org.apache.asterix.app.message.ExecuteStatementRequestMessage.handle(ExecuteStatementRequestMessage.java:181) ~[asterix-app.jar:1.0.0-2203]

	at org.apache.asterix.messaging.CCMessageBroker.receivedMessage(CCMessageBroker.java:64) ~[asterix-app.jar:1.0.0-2203]

	at org.apache.hyracks.control.cc.work.ApplicationMessageWork.lambda$notifyMessageBroker$0(ApplicationMessageWork.java:74) ~[hyracks-control-cc.jar:1.0.0-2203]

	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]

	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]

	at java.base/java.lang.Thread.run(Thread.java:840) [?:?]

2024-07-11T07:09:08.687+00:00 INFO CBAS.server.QueryServiceServlet [HttpExecutor(port:18095)-12] handleRequest: uuid=db4836c0-3564-4e65-8358-ff0406d9b572, clientContextID=null, {"host":"cb.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com:18095","path":"/query/service","statement":"<ud>select value ds.GroupName from Metadata.`Dataset` as ds where ds.DatabaseName not in [\\\"System\\\"] and ds.MetatypeName=\\\"DCPMeta\\\"</ud>","pretty":false,"mode":"immediate","clientContextID":null,"clientType":"ASTERIX","dataverse":null,"format":"CLEAN_JSON","timeout":9223372036854775807,"maxResultReads":1,"planFormat":"JSON","expressionTree":false,"rewrittenExpressionTree":false,"logicalPlan":false,"optimizedLogicalPlan":false,"job":false,"profile":"counts","signature":true,"multiStatement":true,"parseOnly":false,"readOnly":false,"maxWarnings":0,"sqlCompat":false,"source":null,"scanConsistency":null,"scanWait":null}

2024-07-11T07:09:08.699+00:00 INFO CBAS.messaging.NCMessageBroker [Worker:e71ff082812e12d4c853d428a9a7f9c2] Received message: ExecuteStatementResponseMessage(id=332, uuid=db4836c0-3564-4e65-8358-ff0406d9b572, clientContextId=null): 0 characters

2024-07-11T07:09:08.699+00:00 WARN CBAS.server.QueryServiceServlet [HttpExecutor(port:18095)-12] handleException: unexpected exception: uuid=null, clientContextID=db4836c0-3564-4e65-8358-ff0406d9b572

com.couchbase.analytics.common.exceptions.AnalyticsHyracksException: CBAS0022: Internal error

	at com.couchbase.analytics.auth.AuthorizationProviderCloud.ensureAuthorized(AuthorizationProviderCloud.java:112) ~[columnar-server.jar:1.0.0-2203]

	at com.couchbase.analytics.request.AnalyticsReceptionist.ensureSchedulable(AnalyticsReceptionist.java:33) ~[columnar-common.jar:1.0.0-2203]

	at org.apache.asterix.app.translator.QueryTranslator.createAndRunJob(QueryTranslator.java:5423) ~[asterix-app.jar:1.0.0-2203]

	at org.apache.asterix.app.translator.QueryTranslator.deliverResult(QueryTranslator.java:5310) ~[asterix-app.jar:1.0.0-2203]

	at org.apache.asterix.app.translator.QueryTranslator.handleQuery(QueryTranslator.java:5280) ~[asterix-app.jar:1.0.0-2203]

	at org.apache.asterix.app.translator.QueryTranslator.compileAndExecute(QueryTranslator.java:533) ~[asterix-app.jar:1.0.0-2203]

	at org.apache.asterix.app.message.ExecuteStatementRequestMessage.handle(ExecuteStatementRequestMessage.java:181) ~[asterix-app.jar:1.0.0-2203]

	at org.apache.asterix.messaging.CCMessageBroker.receivedMessage(CCMessageBroker.java:64) ~[asterix-app.jar:1.0.0-2203]

	at org.apache.hyracks.control.cc.work.ApplicationMessageWork.lambda$notifyMessageBroker$0(ApplicationMessageWork.java:74) ~[hyracks-control-cc.jar:1.0.0-2203]

	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]

	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]

	at java.base/java.lang.Thread.run(Thread.java:840) [?:?]

2024-07-11T07:03:18.230+00:00 INFO CBAS.messaging.CCMessageBroker [Executor-977:ClusterController] Received message: ExecuteStatementRequestMessage(id=326, from=e71ff082812e12d4c853d428a9a7f9c2, uuid=0f49c576-df9b-4686-9815-4f08dada3111, clientContextID=null): <ud>\nSELECT meta.*\nFROM (\n  SELECT ds.DatabaseName,\n  ds.DataverseName,\nds.DataverseName || '.' || ds.DatasetName AS datasetFullyQualifiedName,\n  ds.DatasetName AS id,\n  TRUE AS isDataset,\n  ds.BucketName AS bucketName,\n  ds.ScopeName AS scopeName,\n  ds.CollectionName AS collectionName,\n  ds.BucketDataverseName AS linkDataverseName,\n  ds.BucketDatabaseName AS linkDatabaseName,\n  ds.LinkDataverseName AS linkDataverseName2,\n  ds.LinkDatabaseName AS linkDatabaseName2,\n  ds.`Filter` AS `filter`,\n  ds.LinkName,\n  ds.DatasetType,\n  ds.DatatypeName,\n  ds.InternalDetails.PrimaryKey as primaryKey,\n  ds.InternalDetails.PrimaryKeyTypes as primaryKeyTypes,\n  ds.InternalDetails.`Autogenerated` as primaryKeyAutogenerated,\n  ds.ViewDetails,\n  t.Derived.Record.Fields AS ViewFields,\n  concat2(', ', (\n    SELECT VALUE FieldName || ' ' || ( CASE WHEN LOWER(FieldType) = 'int64' THEN 'BIGINT' ELSE UPPER(FieldType) END) || ( CASE WHEN NOT IsNullable\nAND NOT IsMissable THEN ' NOT UNKNOWN' ELSE '' END)\nFROM t.Derived.Record.Fields)) AS TypeString,\n  (\n    SELECT idx.IndexName,\n  idx.SearchKey,\n  idx.SearchKeyType\nFROM Metadata.`Index` AS idx\nWHERE idx.IsPrimary = FALSE\nAND idx.DatabaseName = ds.DatabaseName\nAND idx.DatasetName = ds.DatasetName\nAND idx.DataverseName = ds.DataverseName) AS indexes,\n  ds.ExternalDetails.Properties AS externalDetails\nFROM Metadata.`Dataset` AS ds LEFT\nJOIN Metadata.Datatype t ON ds.DatabaseName = t.DatabaseName AND ds.DataverseName = t.DataverseName\nAND t.DatatypeName = ds.DatatypeName\nWHERE (ds.DataverseName != 'Metadata'\n)\nUNION ALL\nSELECT dv.DatabaseName,\n  dv.DataverseName,\n  TRUE AS isDataverse,\n  (\n    SELECT l.Name\nFROM Metadata.`Link` AS l\nWHERE l.DatabaseName = dv.DatabaseName AND l.DataverseName = dv.DataverseName) AS links\nFROM Metadata.`Dataverse` AS dv\nWHERE dv.DataverseName != 'Metadata'\n\n  UNION ALL\nSELECT db.DatabaseName,\n  TRUE AS isDatabase\nFROM Metadata.`Database` AS db\nWHERE db.DatabaseName != 'System'\n\n  UNION ALL\nSELECT DatabaseName,\n  DataverseName,\n  Name,\n  IsActive,\n  ClusterName,\n  `Type` AS LinkType,\n  TRUE AS isLink\nFROM Metadata.`Link`) meta\nORDER BY\n  meta.isDatabase DESC,\n  meta.isDataverse DESC,\n  meta.isLink DESC,\n  meta.DatabaseName,\n  meta.DataverseName,\n  meta.LinkName,\n  meta.id;</ud>

2024-07-11T07:03:18.284+00:00 WARN CBAS.apache.asterix [Executor-977:ClusterController] CBAS0014: Insufficient permissions or the requested object does not exist

com.couchbase.analytics.common.exceptions.AnalyticsHyracksException: CBAS0014: Insufficient permissions or the requested object does not exist

	at com.couchbase.analytics.auth.AuthorizationProviderCloud.ensureAuthorized(AuthorizationProviderCloud.java:115) ~[columnar-server.jar:1.0.0-2203]

	at com.couchbase.analytics.request.AnalyticsReceptionist.ensureSchedulable(AnalyticsReceptionist.java:33) ~[columnar-common.jar:1.0.0-2203]

	at org.apache.asterix.app.translator.QueryTranslator.createAndRunJob(QueryTranslator.java:5423) ~[asterix-app.jar:1.0.0-2203]

	at org.apache.asterix.app.translator.QueryTranslator.deliverResult(QueryTranslator.java:5310) ~[asterix-app.jar:1.0.0-2203]

	at org.apache.asterix.app.translator.QueryTranslator.handleQuery(QueryTranslator.java:5280) ~[asterix-app.jar:1.0.0-2203]

	at org.apache.asterix.app.translator.QueryTranslator.compileAndExecute(QueryTranslator.java:533) ~[asterix-app.jar:1.0.0-2203]

	at org.apache.asterix.app.message.ExecuteStatementRequestMessage.handle(ExecuteStatementRequestMessage.java:181) ~[asterix-app.jar:1.0.0-2203]

	at org.apache.asterix.messaging.CCMessageBroker.receivedMessage(CCMessageBroker.java:64) ~[asterix-app.jar:1.0.0-2203]

	at org.apache.hyracks.control.cc.work.ApplicationMessageWork.lambda$notifyMessageBroker$0(ApplicationMessageWork.java:74) ~[hyracks-control-cc.jar:1.0.0-2203]

	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]

	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]

	at java.base/java.lang.Thread.run(Thread.java:840) [?:?]