diff --git a/src/AWS/Orleans.Clustering.DynamoDB/README.md b/src/AWS/Orleans.Clustering.DynamoDB/README.md index aa522789be2..8a66fa8eef5 100644 --- a/src/AWS/Orleans.Clustering.DynamoDB/README.md +++ b/src/AWS/Orleans.Clustering.DynamoDB/README.md @@ -42,11 +42,9 @@ var builder = Host.CreateApplicationBuilder(args) // Configure DynamoDB clustering .UseDynamoDBClustering(options => { - options.AccessKey = "YOUR_AWS_ACCESS_KEY"; - options.SecretKey = "YOUR_AWS_SECRET_KEY"; - options.Region = "us-east-1"; + options.Service = "us-east-1"; options.TableName = "OrleansClusteringTable"; - options.CreateIfNotExists = true; + options.UseProvisionedThroughput = false; }); }); @@ -65,6 +63,12 @@ Console.WriteLine($"Grain response: {response}"); await host.WaitForShutdownAsync(); ``` +## Endpoints and credentials + +`Service` selects an AWS region or an HTTP/HTTPS endpoint URL. `AccessKey` and `SecretKey` supply explicit credentials; adding `Token` supplies session credentials. `ProfileName` selects a named AWS profile when explicit keys are omitted. These credentials apply to both regions and endpoint URLs. + +The AWS SDK credential chain supplies credentials for regions and HTTPS endpoints when explicit keys and a profile are omitted. HTTP endpoints use the local-development credentials `dummy` and `dummyKey` in that case. Configure explicit credentials when a DynamoDB Local instance uses an access key to select its database. + ## Documentation For more comprehensive documentation, please refer to: - [Microsoft Orleans Documentation](https://dotnet.github.io/orleans/docs/) diff --git a/src/AWS/Orleans.Persistence.DynamoDB/README.md b/src/AWS/Orleans.Persistence.DynamoDB/README.md index ec401857732..e67c597b1b5 100644 --- a/src/AWS/Orleans.Persistence.DynamoDB/README.md +++ b/src/AWS/Orleans.Persistence.DynamoDB/README.md @@ -26,11 +26,9 @@ var builder = Host.CreateApplicationBuilder(args) name: "dynamoStore", configureOptions: options => { - options.AccessKey = "YOUR_AWS_ACCESS_KEY"; - options.SecretKey = "YOUR_AWS_SECRET_KEY"; - options.Region = "us-east-1"; + options.Service = "us-east-1"; options.TableName = "OrleansGrainState"; - options.CreateIfNotExists = true; + options.UseProvisionedThroughput = false; }); }); @@ -38,6 +36,12 @@ var builder = Host.CreateApplicationBuilder(args) await builder.RunAsync(); ``` +## Endpoints and credentials + +`Service` selects an AWS region or an HTTP/HTTPS endpoint URL. `AccessKey` and `SecretKey` supply explicit credentials; adding `Token` supplies session credentials. `ProfileName` selects a named AWS profile when explicit keys are omitted. These credentials apply to both regions and endpoint URLs. + +The AWS SDK credential chain supplies credentials for regions and HTTPS endpoints when explicit keys and a profile are omitted. HTTP endpoints use the local-development credentials `dummy` and `dummyKey` in that case. Configure explicit credentials when a DynamoDB Local instance uses an access key to select its database. + ## Example - Using Grain Storage in a Grain ```csharp // Define grain state class diff --git a/src/AWS/Shared/Storage/DynamoDBStorage.cs b/src/AWS/Shared/Storage/DynamoDBStorage.cs index 2e052d732c5..492b82084d6 100755 --- a/src/AWS/Shared/Storage/DynamoDBStorage.cs +++ b/src/AWS/Shared/Storage/DynamoDBStorage.cs @@ -147,49 +147,56 @@ public async Task InitializeTable(string tableName, List keys, [MemberNotNull(nameof(_ddbClient))] private void CreateClient() { - if (this._service.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || - this._service.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) + var isServiceUrl = Uri.TryCreate(this._service, UriKind.Absolute, out var serviceUri) + && (serviceUri.Scheme == Uri.UriSchemeHttp || serviceUri.Scheme == Uri.UriSchemeHttps); + var config = isServiceUrl + ? new AmazonDynamoDBConfig { ServiceURL = this._service } + : new AmazonDynamoDBConfig { RegionEndpoint = AWSUtils.GetRegionEndpoint(this._service) }; + var credentials = GetClientCredentials(serviceUri); + + this._ddbClient = credentials is null + ? new AmazonDynamoDBClient(config) + : new AmazonDynamoDBClient(credentials, config); + } + + private AWSCredentials? GetClientCredentials(Uri? serviceUri) + { + var credentials = GetExplicitCredentials(); + if (credentials is null && serviceUri?.Scheme == Uri.UriSchemeHttp) { - // Local DynamoDB instance (for testing) - var credentials = new BasicAWSCredentials("dummy", "dummyKey"); - this._ddbClient = new AmazonDynamoDBClient(credentials, new AmazonDynamoDBConfig { ServiceURL = this._service }); + credentials = new BasicAWSCredentials("dummy", "dummyKey"); } - else if (!string.IsNullOrEmpty(this._accessKey) && !string.IsNullOrEmpty(this.secretKey) && !string.IsNullOrEmpty(this._token)) + + return credentials; + } + + private AWSCredentials? GetExplicitCredentials() + { + if (!string.IsNullOrEmpty(this._accessKey) + && !string.IsNullOrEmpty(this.secretKey) + && !string.IsNullOrEmpty(this._token)) { - // AWS DynamoDB instance (auth via explicit credentials and token) - var credentials = new SessionAWSCredentials(this._accessKey, this.secretKey, this._token); - this._ddbClient = new AmazonDynamoDBClient(credentials, new AmazonDynamoDBConfig { RegionEndpoint = AWSUtils.GetRegionEndpoint(this._service) }); + return new SessionAWSCredentials(this._accessKey, this.secretKey, this._token); } - else if (!string.IsNullOrEmpty(this._accessKey) && !string.IsNullOrEmpty(this.secretKey)) + + if (!string.IsNullOrEmpty(this._accessKey) && !string.IsNullOrEmpty(this.secretKey)) { - // AWS DynamoDB instance (auth via explicit credentials) - var credentials = new BasicAWSCredentials(this._accessKey, this.secretKey); - this._ddbClient = new AmazonDynamoDBClient(credentials, new AmazonDynamoDBConfig { RegionEndpoint = AWSUtils.GetRegionEndpoint(this._service) }); + return new BasicAWSCredentials(this._accessKey, this.secretKey); } - else if (!string.IsNullOrEmpty(this._profileName)) + + if (!string.IsNullOrEmpty(this._profileName)) { - // AWS DynamoDB instance (auth via explicit credentials and token found in a named profile) var chain = new CredentialProfileStoreChain(); if (chain.TryGetAWSCredentials(this._profileName, out var credentials)) { - this._ddbClient = new AmazonDynamoDBClient( - credentials, - new AmazonDynamoDBConfig - { - RegionEndpoint = AWSUtils.GetRegionEndpoint(this._service) - }); + return credentials; } - else - { - throw new InvalidOperationException( - $"AWS named profile '{this._profileName}' provided, but credentials could not be retrieved"); - } - } - else - { - // AWS DynamoDB instance (implicit auth - EC2 IAM Roles etc) - this._ddbClient = new AmazonDynamoDBClient(new AmazonDynamoDBConfig { RegionEndpoint = AWSUtils.GetRegionEndpoint(this._service) }); + + throw new InvalidOperationException( + $"AWS named profile '{this._profileName}' provided, but credentials could not be retrieved"); } + + return null; } private async Task GetTableDescription(string tableName, CancellationToken cancellationToken = default) diff --git a/test/Extensions/Orleans.AWS.Tests/StorageTests/DynamoDBStorageCredentialTests.cs b/test/Extensions/Orleans.AWS.Tests/StorageTests/DynamoDBStorageCredentialTests.cs new file mode 100644 index 00000000000..ee4e1540def --- /dev/null +++ b/test/Extensions/Orleans.AWS.Tests/StorageTests/DynamoDBStorageCredentialTests.cs @@ -0,0 +1,241 @@ +using System.Net; +using System.Net.Http.Headers; +using Amazon; +using Amazon.DynamoDBv2; +using Amazon.DynamoDBv2.Model; +using Amazon.Runtime; +using Amazon.Runtime.Credentials; +using Microsoft.Extensions.Logging.Abstractions; +using Orleans.AWSUtils.Tests; +using Xunit; + +namespace AWSUtils.Tests.StorageTests; + +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class DynamoDBCredentialEnvironmentCollection +{ + public const string Name = "DynamoDB credential environment"; +} + +[Collection(DynamoDBCredentialEnvironmentCollection.Name)] +[TestSuite("BVT")] +[TestProvider("DynamoDB")] +[TestArea("Storage")] +[TestCategory("AWS"), TestCategory("DynamoDB"), TestCategory("BVT")] +public sealed class DynamoDBStorageCredentialTests +{ + [Theory] + [InlineData("us-west-2")] + [InlineData("https://dynamodb.example")] + [InlineData("HTTPS://dynamodb.example")] + public void DynamoDBStorage_WithoutExplicitCredentials_PreservesDefaultChain(string service) + { + var storage = new DynamoDBStorage(NullLogger.Instance, service); + using var client = storage.ClientForTest; + + Assert.Null(storage.ExplicitCredentialsForTest); + Assert.IsType( + client.Config.IdentityResolverConfiguration.GetIdentityResolver()); + AssertService(storage, service); + } + + [Theory] + [InlineData("http://dynamodb:8000")] + [InlineData("HTTP://dynamodb:8000")] + public void DynamoDBStorage_HttpEmulatorEndpointWithoutCredentials_UsesDummyCredentials(string service) + { + var storage = new DynamoDBStorage(NullLogger.Instance, service); + using var client = storage.ClientForTest; + + var credentials = Assert.IsType(storage.ExplicitCredentialsForTest).GetCredentials(); + + Assert.Equal("dummy", credentials.AccessKey); + Assert.Equal("dummyKey", credentials.SecretKey); + Assert.Equal(string.Empty, credentials.Token); + AssertService(storage, service); + } + + [Theory] + [InlineData("us-east-2", "us-east-2")] + [InlineData("http://dynamodb:8000", "us-east-1")] + [InlineData("https://dynamodb.example", "us-east-1")] + [InlineData("https://dynamodb.us-west-2.amazonaws.com", "us-west-2")] + [InlineData("https://dynamodb.eu-west-2.amazonaws.com", "eu-west-2")] + [InlineData("https://dynamodb.cn-north-1.amazonaws.com.cn", "cn-north-1")] + [InlineData("https://dynamodb-fips.us-gov-west-1.amazonaws.com", "us-gov-west-1")] + public async Task DynamoDBStorage_ExplicitAccessAndSecret_UsesBasicCredentials(string service, string signingRegion) + { + var storage = new DynamoDBStorage( + NullLogger.Instance, + service, + accessKey: "explicit-access", + secretKey: "explicit-secret", + profileName: $"unused-{Guid.NewGuid():N}"); + using var client = storage.ClientForTest; + + var credentials = Assert.IsType(storage.ExplicitCredentialsForTest).GetCredentials(); + + Assert.Equal("explicit-access", credentials.AccessKey); + Assert.Equal("explicit-secret", credentials.SecretKey); + Assert.Equal(string.Empty, credentials.Token); + AssertService(storage, service); + await AssertRequestSigningAsync(storage, signingRegion, "explicit-access", token: null); + } + + [Theory] + [InlineData("eu-west-2", "eu-west-2")] + [InlineData("http://dynamodb:8000", "us-east-1")] + [InlineData("https://dynamodb.example", "us-east-1")] + [InlineData("https://dynamodb.us-west-2.amazonaws.com", "us-west-2")] + [InlineData("https://dynamodb.eu-west-2.amazonaws.com", "eu-west-2")] + [InlineData("https://dynamodb.cn-north-1.amazonaws.com.cn", "cn-north-1")] + [InlineData("https://dynamodb-fips.us-gov-west-1.amazonaws.com", "us-gov-west-1")] + public async Task DynamoDBStorage_ExplicitSessionCredentials_UsesSessionCredentials(string service, string signingRegion) + { + var storage = new DynamoDBStorage( + NullLogger.Instance, + service, + accessKey: "session-access", + secretKey: "session-secret", + token: "session-token", + profileName: $"unused-{Guid.NewGuid():N}"); + using var client = storage.ClientForTest; + + var credentials = Assert.IsType(storage.ExplicitCredentialsForTest).GetCredentials(); + + Assert.Equal("session-access", credentials.AccessKey); + Assert.Equal("session-secret", credentials.SecretKey); + Assert.Equal("session-token", credentials.Token); + AssertService(storage, service); + await AssertRequestSigningAsync(storage, signingRegion, "session-access", "session-token"); + } + + [Theory] + [InlineData("us-west-2", false, "us-west-2")] + [InlineData("http://dynamodb:8000", false, "us-east-1")] + [InlineData("https://dynamodb.example", false, "us-east-1")] + [InlineData("https://dynamodb.us-west-2.amazonaws.com", false, "us-west-2")] + [InlineData("https://dynamodb.eu-west-2.amazonaws.com", false, "eu-west-2")] + [InlineData("https://dynamodb.cn-north-1.amazonaws.com.cn", false, "cn-north-1")] + [InlineData("https://dynamodb-fips.us-gov-west-1.amazonaws.com", false, "us-gov-west-1")] + [InlineData("us-west-2", true, "us-west-2")] + [InlineData("http://dynamodb:8000", true, "us-east-1")] + [InlineData("https://dynamodb.example", true, "us-east-1")] + [InlineData("https://dynamodb.us-west-2.amazonaws.com", true, "us-west-2")] + [InlineData("https://dynamodb.eu-west-2.amazonaws.com", true, "eu-west-2")] + [InlineData("https://dynamodb.cn-north-1.amazonaws.com.cn", true, "cn-north-1")] + [InlineData("https://dynamodb-fips.us-gov-west-1.amazonaws.com", true, "us-gov-west-1")] + public async Task DynamoDBStorage_ProfileName_UsesIsolatedSharedCredentialsProfile(string service, bool useSessionToken, string signingRegion) + { + var profileName = $"dynamodb-credentials-{Guid.NewGuid():N}"; + var credentialsPath = Path.Combine(Path.GetTempPath(), $"{Guid.NewGuid():N}.credentials"); + var previousProfilesLocation = AWSConfigs.AWSProfilesLocation; + try + { + File.WriteAllText( + credentialsPath, + $"[{profileName}]{Environment.NewLine}" + + $"aws_access_key_id = profile-access{Environment.NewLine}" + + $"aws_secret_access_key = profile-secret{Environment.NewLine}" + + (useSessionToken ? $"aws_session_token = profile-token{Environment.NewLine}" : string.Empty)); + AWSConfigs.AWSProfilesLocation = credentialsPath; + + var storage = new DynamoDBStorage( + NullLogger.Instance, + service, + profileName: profileName); + using var client = storage.ClientForTest; + var credentials = useSessionToken + ? Assert.IsType(storage.ExplicitCredentialsForTest).GetCredentials() + : Assert.IsType(storage.ExplicitCredentialsForTest).GetCredentials(); + + Assert.Equal("profile-access", credentials.AccessKey); + Assert.Equal("profile-secret", credentials.SecretKey); + Assert.Equal(useSessionToken ? "profile-token" : string.Empty, credentials.Token); + AssertService(storage, service); + await AssertRequestSigningAsync(storage, signingRegion, "profile-access", useSessionToken ? "profile-token" : null); + + var missingProfileName = $"{profileName}-missing"; + var exception = Assert.Throws(() => new DynamoDBStorage( + NullLogger.Instance, + service, + profileName: missingProfileName)); + + Assert.Equal( + $"AWS named profile '{missingProfileName}' provided, but credentials could not be retrieved", + exception.Message); + } + finally + { + AWSConfigs.AWSProfilesLocation = previousProfilesLocation; + File.Delete(credentialsPath); + } + } + + private static void AssertService(DynamoDBStorage storage, string service) + { + var config = storage.ClientForTest.Config; + if (Uri.TryCreate(service, UriKind.Absolute, out var uri)) + { + Assert.Equal(uri, new Uri(config.ServiceURL)); + Assert.Null(config.RegionEndpoint); + } + else + { + Assert.Equal(service, config.RegionEndpoint.SystemName); + Assert.Null(config.ServiceURL); + } + } + + private static async Task AssertRequestSigningAsync(DynamoDBStorage storage, string region, string accessKey, string? token) + { + using var handler = new RecordingHttpMessageHandler(); + using var httpClient = new HttpClient(handler); + var config = Assert.IsType(storage.ClientForTest.Config); + config.HttpClientFactory = new TestHttpClientFactory(httpClient); + + await storage.ClientForTest.ListTablesAsync(new ListTablesRequest(), TestContext.Current.CancellationToken); + + Assert.Equal(1, handler.RequestCount); + Assert.NotNull(handler.Authorization); + Assert.NotNull(handler.RequestDate); + Assert.StartsWith( + $"AWS4-HMAC-SHA256 Credential={accessKey}/{handler.RequestDate[..8]}/{region}/dynamodb/aws4_request,", + handler.Authorization); + Assert.Equal(token, handler.SessionToken); + if (config.ServiceURL is { } endpoint) + { + Assert.NotNull(handler.RequestUri); + Assert.Equal(new Uri(endpoint).AbsoluteUri, handler.RequestUri.AbsoluteUri); + } + } + + private sealed class TestHttpClientFactory(HttpClient client) : HttpClientFactory + { + public override HttpClient CreateHttpClient(IClientConfig clientConfig) => client; + public override bool UseSDKHttpClientCaching(IClientConfig clientConfig) => false; + public override bool DisposeHttpClientsAfterUse(IClientConfig clientConfig) => false; + } + + private sealed class RecordingHttpMessageHandler : HttpMessageHandler + { + public int RequestCount { get; private set; } + public Uri? RequestUri { get; private set; } + public string? Authorization { get; private set; } + public string? RequestDate { get; private set; } + public string? SessionToken { get; private set; } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + RequestCount++; + RequestUri = request.RequestUri; + Authorization = Assert.Single(request.Headers.GetValues("Authorization")); + RequestDate = Assert.Single(request.Headers.GetValues("X-Amz-Date")); + SessionToken = request.Headers.TryGetValues("X-Amz-Security-Token", out var tokens) ? Assert.Single(tokens) : null; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("""{"TableNames":[]}""", new MediaTypeHeaderValue("application/x-amz-json-1.0")), + }); + } + } +} diff --git a/test/Extensions/Orleans.AWS.Tests/StorageTests/DynamoDBStorageTestHooks.cs b/test/Extensions/Orleans.AWS.Tests/StorageTests/DynamoDBStorageTestHooks.cs new file mode 100644 index 00000000000..a8b7fe6b8ab --- /dev/null +++ b/test/Extensions/Orleans.AWS.Tests/StorageTests/DynamoDBStorageTestHooks.cs @@ -0,0 +1,16 @@ +using System.Reflection; +using Amazon.DynamoDBv2; +using Amazon.Runtime; + +namespace Orleans.AWSUtils.Tests; + +internal partial class DynamoDBStorage +{ + private static readonly PropertyInfo ExplicitCredentialsProperty = typeof(AmazonServiceClient) + .GetProperty("ExplicitAWSCredentials", BindingFlags.Instance | BindingFlags.NonPublic)!; + + internal AmazonDynamoDBClient ClientForTest => _ddbClient; + + internal AWSCredentials? ExplicitCredentialsForTest + => (AWSCredentials?)ExplicitCredentialsProperty.GetValue(_ddbClient); +} diff --git a/test/Transactions/Orleans.Transactions.DynamoDB.Test/DynamoDBTransactionalStateStorageTests.cs b/test/Transactions/Orleans.Transactions.DynamoDB.Test/DynamoDBTransactionalStateStorageTests.cs index 0e9cf18ae41..4d0b9e42841 100644 --- a/test/Transactions/Orleans.Transactions.DynamoDB.Test/DynamoDBTransactionalStateStorageTests.cs +++ b/test/Transactions/Orleans.Transactions.DynamoDB.Test/DynamoDBTransactionalStateStorageTests.cs @@ -532,9 +532,20 @@ private static AmazonDynamoDBClient CreateDynamoDBClient() if (service.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || service.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) { - return new AmazonDynamoDBClient( - new BasicAWSCredentials("dummy", "dummyKey"), - new AmazonDynamoDBConfig { ServiceURL = service }); + var serviceConfig = new AmazonDynamoDBConfig { ServiceURL = service }; + if (!string.IsNullOrEmpty(AWSTestConstants.DynamoDbAccessKey) + && !string.IsNullOrEmpty(AWSTestConstants.DynamoDbSecretKey)) + { + return new AmazonDynamoDBClient( + new BasicAWSCredentials( + AWSTestConstants.DynamoDbAccessKey, + AWSTestConstants.DynamoDbSecretKey), + serviceConfig); + } + + return service.StartsWith("http://", StringComparison.OrdinalIgnoreCase) + ? new AmazonDynamoDBClient(new BasicAWSCredentials("dummy", "dummyKey"), serviceConfig) + : new AmazonDynamoDBClient(serviceConfig); } var config = new AmazonDynamoDBConfig { RegionEndpoint = RegionEndpoint.GetBySystemName(service) };