diff --git a/eng/versioning/external_dependencies.txt b/eng/versioning/external_dependencies.txt index cae552431453..7b79bcdb9532 100644 --- a/eng/versioning/external_dependencies.txt +++ b/eng/versioning/external_dependencies.txt @@ -380,7 +380,7 @@ cosmos-spark_3-2_org.apache.spark:spark-hive_2.12;3.2.0 cosmos-spark_3-3_org.apache.spark:spark-hive_2.12;3.3.0 cosmos-spark_3-4_org.apache.spark:spark-hive_2.12;3.4.0 cosmos-spark_3-5_org.apache.spark:spark-hive_2.12;3.5.0 -cosmos_org.scala-lang:scala-library;2.12.10 +cosmos_org.scala-lang:scala-library;2.12.19 cosmos_org.scala-lang.modules:scala-java8-compat_2.12;0.8.0 cosmos_io.projectreactor:reactor-scala-extensions_2.12;0.8.0 cosmos_commons-io:commons-io;2.4 diff --git a/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml b/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml index 46d7e2be6d51..ededed015151 100644 --- a/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml @@ -20,8 +20,8 @@ MM-dd-HH-mm-ss 0.17 0.18 - 11 - 11 + 8 + 8 true azure_cosmos_spark_sample @@ -84,7 +84,7 @@ org.scala-lang scala-library - 2.12.10 + 2.12.19 provided @@ -143,6 +143,55 @@ + + com.globalmentor + hadoop-bare-naked-local-fs + 0.1.0 + test + + + org.mockito + mockito-core + 4.8.1 + test + + + io.projectreactor + reactor-test + 3.4.36 + test + + + org.assertj + assertj-core + 3.22.0 + test + + + org.scalatest + scalatest_2.12 + 3.2.2 + test + + + org.scalatest + scalatest-flatspec_2.12 + 3.2.3 + test + + + org.scalactic + scalactic_2.12 + 3.2.3 + test + + + org.scalamock + scalamock_2.12 + 5.0.0 + test + + org.apache.spark:spark-sql_2.12:[3.5.0] commons-io:commons-io:[2.4] - org.scala-lang:scala-library:[2.12.10] + org.scala-lang:scala-library:[2.12.19] org.scala-lang.modules:scala-java8-compat_2.12:[0.8.0] io.projectreactor:reactor-scala-extensions_2.12:[0.8.0] org.scalatest:scalatest_2.12:[3.2.2] @@ -325,9 +374,9 @@ scala-maven-plugin 4.8.1 - 11 - 11 - 2.12.10 + 1.8 + 1.8 + 2.12.19 @@ -738,13 +787,13 @@ default-compile - 11 + 8 base-compile @@ -752,7 +801,7 @@ compile - 11 + 8 @@ -761,8 +810,8 @@ testCompile - 11 - 11 + 8 + 8 @@ -771,7 +820,7 @@ testCompile - 11 + 8 diff --git a/sdk/cosmos/azure-cosmos-spark_3_2-12/pom.xml b/sdk/cosmos/azure-cosmos-spark_3_2-12/pom.xml index eb3f0fa62bd2..9d3388c67b9b 100644 --- a/sdk/cosmos/azure-cosmos-spark_3_2-12/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3_2-12/pom.xml @@ -20,8 +20,8 @@ MM-dd-HH-mm-ss 0.17 0.18 - 11 - 11 + 8 + 8 true azure_cosmos_spark azurecosmosspark @@ -47,12 +47,11 @@ - org.scala-lang scala-library - 2.12.10 + 2.12.19 provided @@ -228,7 +227,7 @@ org.apache.spark:spark-sql_2.12:[3.4.0] org.apache.spark:spark-sql_2.12:[3.5.0] commons-io:commons-io:[2.4] - org.scala-lang:scala-library:[2.12.10] + org.scala-lang:scala-library:[2.12.19] org.scala-lang.modules:scala-java8-compat_2.12:[0.8.0] io.projectreactor:reactor-scala-extensions_2.12:[0.8.0] org.scalatest:scalatest_2.12:[3.2.2] @@ -326,9 +325,9 @@ scala-maven-plugin 4.8.1 - 11 - 11 - 2.12.10 + 1.8 + 1.8 + 2.12.19 @@ -768,13 +767,13 @@ default-compile - 11 + 8 base-compile @@ -782,7 +781,7 @@ compile - 11 + 8 @@ -791,8 +790,8 @@ testCompile - 11 - 11 + 8 + 8 @@ -801,7 +800,7 @@ testCompile - 11 + 8 diff --git a/sdk/cosmos/azure-cosmos-spark_3_2-12/src/main/scala/com/azure/cosmos/spark/CosmosClientCache.scala b/sdk/cosmos/azure-cosmos-spark_3_2-12/src/main/scala/com/azure/cosmos/spark/CosmosClientCache.scala index 1ba20f0e96df..b4ad5cc0caa0 100644 --- a/sdk/cosmos/azure-cosmos-spark_3_2-12/src/main/scala/com/azure/cosmos/spark/CosmosClientCache.scala +++ b/sdk/cosmos/azure-cosmos-spark_3_2-12/src/main/scala/com/azure/cosmos/spark/CosmosClientCache.scala @@ -17,6 +17,7 @@ import org.apache.spark.scheduler.{SparkListener, SparkListenerApplicationEnd} import org.apache.spark.sql.SparkSession import org.apache.spark.{SparkContext, TaskContext} import reactor.core.publisher.Mono +import reactor.core.scheduler.{Scheduler, Schedulers} import java.time.{Duration, Instant} import java.util.ConcurrentModificationException @@ -52,6 +53,15 @@ private[spark] object CosmosClientCache extends BasicLoggingTrait { this.cleanupIntervalInSeconds, TimeUnit.SECONDS) + private val AAD_AUTH_BOUNDED_ELASTIC_THREAD_NAME = "cosmos-client-cache-auth-bounded-elastic" + private val TTL_FOR_SCHEDULER_WORKER_IN_SECONDS = 60 // same as BoundedElasticScheduler.DEFAULT_TTL_SECONDS + + private val aadAuthBoundedElastic: Scheduler = Schedulers.newBoundedElastic( + Schedulers.DEFAULT_BOUNDED_ELASTIC_SIZE, + Schedulers.DEFAULT_BOUNDED_ELASTIC_QUEUESIZE, + AAD_AUTH_BOUNDED_ELASTIC_THREAD_NAME, + TTL_FOR_SCHEDULER_WORKER_IN_SECONDS, true) + def apply(cosmosClientConfiguration: CosmosClientConfiguration, cosmosClientStateHandle: Option[CosmosClientMetadataCachesSnapshot], calledFrom: String): CosmosClientCacheItem = { @@ -615,10 +625,14 @@ private[spark] object CosmosClientCache extends BasicLoggingTrait { private[this] class CosmosAccessTokenCredential(val tokenProvider: (List[String]) =>CosmosAccessToken) extends TokenCredential { override def getToken(tokenRequestContext: TokenRequestContext): Mono[AccessToken] = { - val token = tokenProvider - .apply(tokenRequestContext.getScopes.asScala.toList) + val returnValue: Mono[AccessToken] = Mono.fromCallable(() => { + val token = tokenProvider + .apply(tokenRequestContext.getScopes.asScala.toList) + + new AccessToken(token.token, token.Offset) + }) - Mono.just(new AccessToken(token.token, token.Offset)) + returnValue.publishOn(aadAuthBoundedElastic) } } }