Skip to content

fix for cosmos spark preferred regions issue #27084

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
FabianMeiswinkel merged 9 commits into from
Feb 15, 2022
Merged

fix for cosmos spark preferred regions issue #27084

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
d66a2ba
added a comment on query plan caching.
RaviTella Sep 13, 2021
7e8c968
Merge pull request #1 from RaviTella/cosmos_spring_readme_update
RaviTella Sep 13, 2021
53fe8b6
Merge remote-tracking branch 'upstream/main' into main
RaviTella Oct 11, 2021
c863c62
Merge remote-tracking branch 'upstream/main' into main
RaviTella Jan 20, 2022
a9cd714
Merge branch 'main' of https://github.com/Azure/azure-sdk-for-java in…
RaviTella Jan 22, 2022
6d03718
fix for cosmos spark preferred regions issue
RaviTella Feb 14, 2022
94d2fc9
fix for cosmos spark preferred regions issue
RaviTella Feb 15, 2022
f44661f
Merge remote-tracking branch 'origin/master' into users/ratella/spark…
RaviTella Feb 15, 2022
6e4b863
Merge branch 'Azure:main' into users/ratella/sparkpreferredlocations
RaviTella Feb 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions ...cosmos/azure-cosmos-spark_3_2-12/src/main/scala/com/azure/cosmos/spark/CosmosConfig.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,7 @@ private object CosmosAccountConfig {
helpMessage = "Cosmos DB Account Name")


private val PreferredRegionRegex = "^[a-z0-9]+$"r // this is for the final form after lower-casing and trimming the whitespaces
private val PreferredRegionRegex = "^[a-z0-9\\d]+(?: [a-z0-9\\d]+)*$".r
private val PreferredRegionsList = CosmosConfigEntry[Array[String]](key = CosmosConfigNames.PreferredRegionsList,
Option.apply(CosmosConfigNames.PreferredRegions),
mandatory = false,
Expand All @@ -268,7 +268,7 @@ private object CosmosAccountConfig {
} else {
trimmedInput.split(",")
.toStream
.map(preferredRegion => preferredRegion.toLowerCase(Locale.ROOT).replace(" ", ""))
.map(preferredRegion => preferredRegion.toLowerCase(Locale.ROOT).trim)
.map(preferredRegion => {
if (!PreferredRegionRegex.findFirstIn(preferredRegion).isDefined) {
throw new IllegalArgumentException(s"$preferredRegionsListAsString is invalid")
Expand Down
41 changes: 33 additions & 8 deletions ...os/azure-cosmos-spark_3_2-12/src/test/scala/com/azure/cosmos/spark/CosmosConfigSpec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ class CosmosConfigSpec extends UnitSpec {
endpointConfig.applicationName.get shouldEqual "myapp"
endpointConfig.useGatewayMode shouldEqual true
endpointConfig.preferredRegionsList.isDefined shouldEqual true
endpointConfig.preferredRegionsList.get should contain theSameElementsAs Array("westus", "eastus1")
endpointConfig.preferredRegionsList.get should contain theSameElementsAs Array("west us", "eastus1")
}

"Config Parser" should "parse account credentials with spark.cosmos.preferredRegions" in {
Expand All @@ -46,7 +46,7 @@ class CosmosConfigSpec extends UnitSpec {
endpointConfig.applicationName.get shouldEqual "myapp"
endpointConfig.useGatewayMode shouldEqual true
endpointConfig.preferredRegionsList.isDefined shouldEqual true
endpointConfig.preferredRegionsList.get should contain theSameElementsAs Array("westus", "eastus1")
endpointConfig.preferredRegionsList.get should contain theSameElementsAs Array("west us", "eastus1")
}

"Config Parser" should "parse account credentials with spark.cosmos.preferredRegions and spark.cosmos.preferredRegionsList" in {
Expand Down Expand Up @@ -116,6 +116,22 @@ class CosmosConfigSpec extends UnitSpec {
"invalid configuration for spark.cosmos.preferredRegionsList:[westus, eastus. Config description: Preferred Region List"
}
}

userConfig = Map(
"spark.cosmos.accountEndpoint" -> sampleProdEndpoint,
"spark.cosmos.accountKey" -> "xyz",
"spark.cosmos.preferredRegionsList" -> "[west us, eastus]"
)

try {
CosmosAccountConfig.parseCosmosAccountConfig(userConfig)
fail("invalid preferred region list")
} catch {
case e: Exception => {
e.getMessage shouldEqual
"invalid configuration for spark.cosmos.preferredRegionsList:[west us, eastus]. Config description: Preferred Region List"
}
}
}

it should "preferred regions parsing" in {
Expand All @@ -126,7 +142,7 @@ class CosmosConfigSpec extends UnitSpec {
)

var config = CosmosAccountConfig.parseCosmosAccountConfig(userConfig)
config.preferredRegionsList.get should contain theSameElementsAs Array("eastus", "westus1")
config.preferredRegionsList.get should contain theSameElementsAs Array("eastus", "west us1")

userConfig = Map(
"spark.cosmos.accountEndpoint" -> sampleProdEndpoint,
Expand All @@ -135,7 +151,7 @@ class CosmosConfigSpec extends UnitSpec {
)

config = CosmosAccountConfig.parseCosmosAccountConfig(userConfig)
config.preferredRegionsList.get should contain theSameElementsAs Array("eastus", "westus1")
config.preferredRegionsList.get should contain theSameElementsAs Array("eastus", "west us1")

userConfig = Map(
"spark.cosmos.accountEndpoint" -> sampleProdEndpoint,
Expand All @@ -144,7 +160,7 @@ class CosmosConfigSpec extends UnitSpec {
)

config = CosmosAccountConfig.parseCosmosAccountConfig(userConfig)
config.preferredRegionsList.get should contain theSameElementsAs Array("eastus", "westus1")
config.preferredRegionsList.get should contain theSameElementsAs Array("eastus", "west us1")

userConfig = Map(
"spark.cosmos.accountEndpoint" -> sampleProdEndpoint,
Expand All @@ -153,7 +169,7 @@ class CosmosConfigSpec extends UnitSpec {
)

config = CosmosAccountConfig.parseCosmosAccountConfig(userConfig)
config.preferredRegionsList.get should contain theSameElementsAs Array("eastus", "westus1")
config.preferredRegionsList.get should contain theSameElementsAs Array("eastus", "west us1")

userConfig = Map(
"spark.cosmos.accountEndpoint" -> sampleProdEndpoint,
Expand All @@ -162,7 +178,7 @@ class CosmosConfigSpec extends UnitSpec {
)

config = CosmosAccountConfig.parseCosmosAccountConfig(userConfig)
config.preferredRegionsList.get should contain theSameElementsAs Array("westus1")
config.preferredRegionsList.get should contain theSameElementsAs Array("west us1")

userConfig = Map(
"spark.cosmos.accountEndpoint" -> sampleProdEndpoint,
Expand All @@ -171,7 +187,7 @@ class CosmosConfigSpec extends UnitSpec {
)

config = CosmosAccountConfig.parseCosmosAccountConfig(userConfig)
config.preferredRegionsList.get should contain theSameElementsAs Array("westus1")
config.preferredRegionsList.get should contain theSameElementsAs Array("west us1")

userConfig = Map(
"spark.cosmos.accountEndpoint" -> sampleProdEndpoint,
Expand All @@ -182,6 +198,15 @@ class CosmosConfigSpec extends UnitSpec {
config = CosmosAccountConfig.parseCosmosAccountConfig(userConfig)
config.preferredRegionsList.get should contain theSameElementsAs Array[String]()

userConfig = Map(
"spark.cosmos.accountEndpoint" -> sampleProdEndpoint,
"spark.cosmos.accountKey" -> "xyz",
"spark.cosmos.preferredRegionsList" -> "[west us 1, east us 2]"
)

config = CosmosAccountConfig.parseCosmosAccountConfig(userConfig)
config.preferredRegionsList.get should contain theSameElementsAs Array("west us 1","east us 2")

}


Expand Down