Details
-
Improvement
-
Resolution: Fixed
-
Major
-
None
-
None
-
None
-
0
-
SDK36
Description
see https://issues.couchbase.com/browse/CBSE-15215
The issue is with the filtering of properties in CouchbaseConfig. It filters out all spark.couchbase.<property> and all the spark.couchbase.<property>:<connectionIdentifier> properties.
So when you have a connection-identified properties, you must select that connectionIdentifier, else they won't be filtered out, and the propertyloader will try to load them. Also, if you added a second set of connection-identified properties, if you selected on the first connection-identifier, then the other's properties would not be filtered.
val filteredProperties = properties
|
.filterKeys(key => {
|
val prefixedKey = PREFIX + key
|
prefixedKey != ident(USERNAME, connectionIdentifier) &&
|
prefixedKey != USERNAME &&
|
prefixedKey != ident(PASSWORD, connectionIdentifier) &&
|
prefixedKey != PASSWORD &&
|
prefixedKey != ident(CONNECTION_STRING, connectionIdentifier) &&
|
prefixedKey != CONNECTION_STRING &&
|
prefixedKey != ident(BUCKET_NAME, connectionIdentifier) &&
|
prefixedKey != BUCKET_NAME &&
|
prefixedKey != ident(SCOPE_NAME, connectionIdentifier) &&
|
prefixedKey != SCOPE_NAME &&
|
prefixedKey != ident(COLLECTION_NAME, connectionIdentifier) &&
|
prefixedKey != COLLECTION_NAME &&
|
prefixedKey != ident(WAIT_UNTIL_READY_TIMEOUT, connectionIdentifier) &&
|
prefixedKey != WAIT_UNTIL_READY_TIMEOUT &&
|
prefixedKey != ident(SPARK_SSL_ENABLED, connectionIdentifier) &&
|
prefixedKey != SPARK_SSL_ENABLED &&
|
prefixedKey != ident(SPARK_SSL_KEYSTORE, connectionIdentifier) &&
|
prefixedKey != SPARK_SSL_KEYSTORE &&
|
prefixedKey != ident(SPARK_SSL_KEYSTORE_PASSWORD, connectionIdentifier) &&
|
prefixedKey != SPARK_SSL_KEYSTORE_PASSWORD
|
})
|
.map(kv => {
|
(kv._1, kv._2)
|
})
|
.toSeq
|
The resulting exception is :
Exception in thread "main" com.couchbase.client.core.env.InvalidPropertyException: Failed to apply environment config property 'password:scenario2' with value 'pass123'. No one-arg setter for property "password:scenario2" in class com.couchbase.client.scala.env.CoreEnvironmentWrapper
|
at com.couchbase.client.core.env.InvalidPropertyException.forProperty(InvalidPropertyException.java:36)
|
at com.couchbase.client.core.env.BuilderPropertySetter.set(BuilderPropertySetter.java:107)
|
at com.couchbase.client.core.env.AbstractMapPropertyLoader.lambda$load$0(AbstractMapPropertyLoader.java:44)
|
at java.util.Map.forEach(Map.java:630)
|
at com.couchbase.client.core.env.AbstractMapPropertyLoader.load(AbstractMapPropertyLoader.java:44)
|
at com.couchbase.client.core.env.CoreEnvironment$Builder.load(CoreEnvironment.java:761)
|
at com.couchbase.client.scala.env.ClusterEnvironment.$anonfun$new$15(ClusterEnvironment.scala:346)
|
at scala.collection.immutable.List.foreach(List.scala:431)
|
at com.couchbase.client.scala.env.ClusterEnvironment.<init>(ClusterEnvironment.scala:346)
|
at com.couchbase.client.scala.env.ClusterEnvironment$Builder.$anonfun$build$1(ClusterEnvironment.scala:86)
|
at scala.util.Try$.apply(Try.scala:213)
|
at com.couchbase.client.scala.env.ClusterEnvironment$Builder.build(ClusterEnvironment.scala:86)
|
at com.couchbase.spark.config.CouchbaseConnection.cluster(CouchbaseConnection.scala:90)
|
at com.couchbase.spark.query.QueryTableProvider.inferSchema(QueryTableProvider.scala:106)
|
at org.apache.spark.sql.execution.datasources.v2.DataSourceV2Utils$.getTableFromProvider(DataSourceV2Utils.scala:90)
|
at org.apache.spark.sql.execution.datasources.v2.DataSourceV2Utils$.loadV2Source(DataSourceV2Utils.scala:140)
|
at org.apache.spark.sql.DataFrameReader.$anonfun$load$1(DataFrameReader.scala:209)
|
at scala.Option.flatMap(Option.scala:271)
|
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:207)
|
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:171)
|
at Main$.main(Main.scala:21)
|
at Main.main(Main.scala)
|
Caused by: com.couchbase.client.core.error.InvalidArgumentException: No one-arg setter for property "password:scenario2" in class com.couchbase.client.scala.env.CoreEnvironmentWrapper
|
at com.couchbase.client.core.error.InvalidArgumentException.fromMessage(InvalidArgumentException.java:28)
|
at com.couchbase.client.core.env.BuilderPropertySetter.set(BuilderPropertySetter.java:79)
|
... 20 more
|