From e46018da76da73f8cd743538b1e7590a62ce0f8a Mon Sep 17 00:00:00 2001 From: Brennan Conroy Date: Tue, 7 Jan 2020 14:47:05 -0800 Subject: [PATCH 01/15] wip --- eng/Dependencies.props | 1 + eng/ProjectReferences.props | 1 + eng/Versions.props | 1 + .../src/AzureBlobXmlRepository.cs | 290 ++++++++++++++++++ .../AzureDataProtectionBuilderExtensions.cs | 102 ++++++ ...e.DataProtection.Azure.Storage.Blob.csproj | 22 ++ .../test/AzureBlobXmlRepositoryTests.cs | 149 +++++++++ ...zureDataProtectionBuilderExtensionsTest.cs | 32 ++ ...Protection.Azure.Storage.Blob.Tests.csproj | 20 ++ src/DataProtection/DataProtection.sln | 32 ++ 10 files changed, 650 insertions(+) create mode 100644 src/DataProtection/Azure.Storage.Blob/src/AzureBlobXmlRepository.cs create mode 100644 src/DataProtection/Azure.Storage.Blob/src/AzureDataProtectionBuilderExtensions.cs create mode 100644 src/DataProtection/Azure.Storage.Blob/src/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj create mode 100644 src/DataProtection/Azure.Storage.Blob/test/AzureBlobXmlRepositoryTests.cs create mode 100644 src/DataProtection/Azure.Storage.Blob/test/AzureDataProtectionBuilderExtensionsTest.cs create mode 100644 src/DataProtection/Azure.Storage.Blob/test/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.Tests.csproj diff --git a/eng/Dependencies.props b/eng/Dependencies.props index 8e7bb0ecd50f..647043d7c413 100644 --- a/eng/Dependencies.props +++ b/eng/Dependencies.props @@ -163,6 +163,7 @@ and are generated based on the last package release. + diff --git a/eng/ProjectReferences.props b/eng/ProjectReferences.props index dedf038e0b51..c1a4cf7ff36b 100644 --- a/eng/ProjectReferences.props +++ b/eng/ProjectReferences.props @@ -63,6 +63,7 @@ + diff --git a/eng/Versions.props b/eng/Versions.props index ac32a23cfdbc..b7ac67bccb7b 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -233,6 +233,7 @@ 2.2.0 0.9.9 + 12.1.0 0.10.13 4.2.1 4.2.1 diff --git a/src/DataProtection/Azure.Storage.Blob/src/AzureBlobXmlRepository.cs b/src/DataProtection/Azure.Storage.Blob/src/AzureBlobXmlRepository.cs new file mode 100644 index 000000000000..b401146450d1 --- /dev/null +++ b/src/DataProtection/Azure.Storage.Blob/src/AzureBlobXmlRepository.cs @@ -0,0 +1,290 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.IO; +using System.Linq; +using System.Runtime.ExceptionServices; +using System.Threading; +using System.Threading.Tasks; +using System.Xml; +using System.Xml.Linq; +using Microsoft.AspNetCore.DataProtection.Repositories; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using Azure; + +namespace Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob +{ + /// + /// An which is backed by Azure Blob Storage. + /// + /// + /// Instances of this type are thread-safe. + /// + public sealed class AzureBlobXmlRepository : IXmlRepository + { + private const int ConflictMaxRetries = 5; + private static readonly TimeSpan ConflictBackoffPeriod = TimeSpan.FromMilliseconds(200); + + private static readonly XName RepositoryElementName = "repository"; + + private readonly Func _blobRefFactory; + private readonly Random _random; + private BlobData _cachedBlobData; + + /// + /// Creates a new instance of the . + /// + /// A factory which can create + /// instances. The factory must be thread-safe for invocation by multiple + /// concurrent threads, and each invocation must return a new object. + public AzureBlobXmlRepository(Func blobRefFactory) + { + _blobRefFactory = blobRefFactory ?? throw new ArgumentNullException(nameof(blobRefFactory)); + _random = new Random(); + } + + /// + public IReadOnlyCollection GetAllElements() + { + var blobRef = CreateFreshBlobRef(); + + // Shunt the work onto a ThreadPool thread so that it's independent of any + // existing sync context or other potentially deadlock-causing items. + + var elements = Task.Run(() => GetAllElementsAsync(blobRef)).GetAwaiter().GetResult(); + return new ReadOnlyCollection(elements); + } + + /// + public void StoreElement(XElement element, string friendlyName) + { + if (element == null) + { + throw new ArgumentNullException(nameof(element)); + } + + var blobRef = CreateFreshBlobRef(); + + // Shunt the work onto a ThreadPool thread so that it's independent of any + // existing sync context or other potentially deadlock-causing items. + + Task.Run(() => StoreElementAsync(blobRef, element)).GetAwaiter().GetResult(); + } + + private XDocument CreateDocumentFromBlob(byte[] blob) + { + using (var memoryStream = new MemoryStream(blob)) + { + var xmlReaderSettings = new XmlReaderSettings() + { + DtdProcessing = DtdProcessing.Prohibit, IgnoreProcessingInstructions = true + }; + + using (var xmlReader = XmlReader.Create(memoryStream, xmlReaderSettings)) + { + return XDocument.Load(xmlReader); + } + } + } + + private BlobClient CreateFreshBlobRef() + { + // ICloudBlob instances aren't thread-safe, so we need to make sure we're working + // with a fresh instance that won't be mutated by another thread. + + var blobRef = _blobRefFactory(); + if (blobRef == null) + { + throw new InvalidOperationException("The ICloudBlob factory method returned null."); + } + + return blobRef; + } + + private async Task> GetAllElementsAsync(BlobClient blobRef) + { + var data = await GetLatestDataAsync(blobRef); + + if (data == null) + { + // no data in blob storage + return new XElement[0]; + } + + // The document will look like this: + // + // + // + // + // ... + // + // + // We want to return the first-level child elements to our caller. + + var doc = CreateDocumentFromBlob(data.BlobContents); + return doc.Root.Elements().ToList(); + } + + private async Task GetLatestDataAsync(BlobClient blobRef) + { + // Set the appropriate AccessCondition based on what we believe the latest + // file contents to be, then make the request. + + var latestCachedData = Volatile.Read(ref _cachedBlobData); // local ref so field isn't mutated under our feet + var requestCondition = (latestCachedData != null) + ? new BlobRequestConditions() { IfNoneMatch = latestCachedData.ETag } + : null; + + try + { + using (var memoryStream = new MemoryStream()) + { + await blobRef.DownloadToAsync( + destination: memoryStream, + conditions: requestCondition); + + // At this point, our original cache either didn't exist or was outdated. + // We'll update it now and return the updated value; + + var props = await blobRef.GetPropertiesAsync(); + + latestCachedData = new BlobData() + { + BlobContents = memoryStream.ToArray(), + ETag = props.Value.ETag + }; + + } + Volatile.Write(ref _cachedBlobData, latestCachedData); + } + catch (RequestFailedException ex) when (ex.Status == 304) + { + // 304 Not Modified + // Thrown when we already have the latest cached data. + // This isn't an error; we'll return our cached copy of the data. + } + catch (RequestFailedException ex) when (ex.Status == 404) + { + // 404 Not Found + // Thrown when no file exists in storage. + // This isn't an error; we'll delete our cached copy of data. + + latestCachedData = null; + Volatile.Write(ref _cachedBlobData, latestCachedData); + } + + return latestCachedData; + } + + private int GetRandomizedBackoffPeriod() + { + // returns a TimeSpan in the range [0.8, 1.0) * ConflictBackoffPeriod + // not used for crypto purposes + var multiplier = 0.8 + (_random.NextDouble() * 0.2); + return (int) (multiplier * ConflictBackoffPeriod.Ticks); + } + + private async Task StoreElementAsync(BlobClient blobRef, XElement element) + { + // holds the last error in case we need to rethrow it + ExceptionDispatchInfo lastError = null; + + for (var i = 0; i < ConflictMaxRetries; i++) + { + if (i > 1) + { + // If multiple conflicts occurred, wait a small period of time before retrying + // the operation so that other writers can make forward progress. + await Task.Delay(GetRandomizedBackoffPeriod()); + } + + if (i > 0) + { + // If at least one conflict occurred, make sure we have an up-to-date + // view of the blob contents. + await GetLatestDataAsync(blobRef); + } + + // Merge the new element into the document. If no document exists, + // create a new default document and inject this element into it. + + var latestData = Volatile.Read(ref _cachedBlobData); + var doc = (latestData != null) + ? CreateDocumentFromBlob(latestData.BlobContents) + : new XDocument(new XElement(RepositoryElementName)); + doc.Root.Add(element); + + // Turn this document back into a byte[]. + + var serializedDoc = new MemoryStream(); + doc.Save(serializedDoc, SaveOptions.DisableFormatting); + + // Generate the appropriate precondition header based on whether or not + // we believe data already exists in storage. + + BlobRequestConditions requestConditions; + BlobHttpHeaders headers = null; + if (latestData != null) + { + var props = await blobRef.GetPropertiesAsync(); + requestConditions = new BlobRequestConditions() { IfMatch = props.Value.ETag }; + } + else + { + requestConditions = new BlobRequestConditions() { IfNoneMatch = ETag.All }; + // set content type on first write + headers = new BlobHttpHeaders() { ContentType = "application/xml; charset=utf-8" }; + } + + try + { + // Send the request up to the server. + var response = await blobRef.UploadAsync( + serializedDoc, + httpHeaders: headers, + conditions: requestConditions); + + // If we got this far, success! + // We can update the cached view of the remote contents. + + Volatile.Write(ref _cachedBlobData, new BlobData() + { + BlobContents = serializedDoc.ToArray(), + ETag = response.Value.ETag // was updated by Upload routine + }); + + return; + } + catch (RequestFailedException ex) + when (ex.Status == 409 || ex.Status == 412) + { + // 409 Conflict + // This error is rare but can be thrown in very special circumstances, + // such as if the blob in the process of being created. We treat it + // as equivalent to 412 for the purposes of retry logic. + + // 412 Precondition Failed + // We'll get this error if another writer updated the repository and we + // have an outdated view of its contents. If this occurs, we'll just + // refresh our view of the remote contents and try again up to the max + // retry limit. + + lastError = ExceptionDispatchInfo.Capture(ex); + } + } + + // if we got this far, something went awry + lastError.Throw(); + } + + private sealed class BlobData + { + internal byte[] BlobContents; + internal ETag ETag; + } + } +} diff --git a/src/DataProtection/Azure.Storage.Blob/src/AzureDataProtectionBuilderExtensions.cs b/src/DataProtection/Azure.Storage.Blob/src/AzureDataProtectionBuilderExtensions.cs new file mode 100644 index 000000000000..4e116112792b --- /dev/null +++ b/src/DataProtection/Azure.Storage.Blob/src/AzureDataProtectionBuilderExtensions.cs @@ -0,0 +1,102 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Azure.Storage; +using Azure.Storage.Blobs; +using Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob; +using Microsoft.AspNetCore.DataProtection.KeyManagement; +using Microsoft.Extensions.DependencyInjection; + +namespace Microsoft.AspNetCore.DataProtection +{ + /// + /// Contains Azure-specific extension methods for modifying a + /// . + /// + public static class AzureDataProtectionBuilderExtensions + { + /// + /// Configures the data protection system to persist keys to the specified path + /// in Azure Blob Storage. + /// + /// The builder instance to modify. + /// The full URI where the key file should be stored. + /// The URI must contain the SAS token as a query string parameter. + /// The value . + /// + /// The container referenced by must already exist. + /// + public static IDataProtectionBuilder PersistKeysToAzureBlobStorage(this IDataProtectionBuilder builder, Uri blobUri) + { + if (builder == null) + { + throw new ArgumentNullException(nameof(builder)); + } + if (blobUri == null) + { + throw new ArgumentNullException(nameof(blobUri)); + } + + var uriBuilder = new BlobUriBuilder(blobUri); + + // The SAS token is present in the query string. + + if (string.IsNullOrEmpty(uriBuilder.Query)) + { + throw new ArgumentException( + message: "URI does not have a SAS token in the query string.", + paramName: nameof(blobUri)); + } + + var credentials = new StorageSharedKeyCredential(uriBuilder.AccountName, uriBuilder.Query); + var blobName = uriBuilder.BlobName; + uriBuilder.Query = null; // no longer needed + var blobAbsoluteUri = uriBuilder.ToUri(); + + var client = new BlobContainerClient(blobAbsoluteUri, credentials); + + return PersistKeystoAzureBlobStorageInternal(builder, () => client.GetBlobClient(blobName)); + } + + /// + /// Configures the data protection system to persist keys to the specified path + /// in Azure Blob Storage. + /// + /// The builder instance to modify. + /// The in which the + /// key file should be stored. + /// The name of the key file, generally specified + /// as "[subdir/]keys.xml" + /// The value . + /// + /// The container referenced by must already exist. + /// + public static IDataProtectionBuilder PersistKeysToAzureBlobStorage(this IDataProtectionBuilder builder, BlobContainerClient container, string blobName) + { + if (builder == null) + { + throw new ArgumentNullException(nameof(builder)); + } + if (container == null) + { + throw new ArgumentNullException(nameof(container)); + } + if (blobName == null) + { + throw new ArgumentNullException(nameof(blobName)); + } + return PersistKeystoAzureBlobStorageInternal(builder, () => container.GetBlobClient(blobName)); + } + + // important: the Func passed into this method must return a new instance with each call + private static IDataProtectionBuilder PersistKeystoAzureBlobStorageInternal(IDataProtectionBuilder builder, Func blobRefFactory) + { + builder.Services.Configure(options => + { + options.XmlRepository = new AzureBlobXmlRepository(blobRefFactory); + }); + return builder; + } + } +} diff --git a/src/DataProtection/Azure.Storage.Blob/src/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj b/src/DataProtection/Azure.Storage.Blob/src/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj new file mode 100644 index 000000000000..78a7b69ea7a2 --- /dev/null +++ b/src/DataProtection/Azure.Storage.Blob/src/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj @@ -0,0 +1,22 @@ + + + + Microsoft Azure Blob storage support as key store. + netstandard2.0 + true + true + aspnetcore;dataprotection;azure;blob + true + true + + + + + + + + + + + + diff --git a/src/DataProtection/Azure.Storage.Blob/test/AzureBlobXmlRepositoryTests.cs b/src/DataProtection/Azure.Storage.Blob/test/AzureBlobXmlRepositoryTests.cs new file mode 100644 index 000000000000..cabca9a9887b --- /dev/null +++ b/src/DataProtection/Azure.Storage.Blob/test/AzureBlobXmlRepositoryTests.cs @@ -0,0 +1,149 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Xml.Linq; +using Azure; +using Azure.Storage; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using Moq; +using Xunit; + +namespace Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.Test +{ + public class AzureBlobXmlRepositoryTests + { + [Fact] + public void StoreCreatesBlobWhenNotExist() + { + BlobRequestConditions uploadConditions = null; + byte[] bytes = null; + string contentType = null; + + var mock = new Mock(); + + mock.Setup(c => c.UploadAsync( + It.IsAny(), + It.IsAny(), + It.IsAny>(), + It.IsAny(), + It.IsAny>(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(async (Stream strm, BlobHttpHeaders headers, IDictionary metaData, BlobRequestConditions conditions, IProgress progress, AccessTier? access, StorageTransferOptions transfer, CancellationToken token) => + { + using var memoryStream = new MemoryStream(); + strm.CopyTo(memoryStream); + bytes = memoryStream.ToArray(); + uploadConditions = conditions; + contentType = headers?.ContentType; + + await Task.Yield(); + + var mockResponse = new Mock>(); + var mockContentInfo = new Mock(); + mockContentInfo.Setup(c => c.ETag).Returns(ETag.All); + mockResponse.Setup(c => c.Value).Returns(mockContentInfo.Object); + return mockResponse.Object; + }); + + var repository = new AzureBlobXmlRepository(() => mock.Object); + repository.StoreElement(new XElement("Element"), null); + + Assert.Equal("*", uploadConditions.IfNoneMatch.ToString()); + Assert.Equal("application/xml; charset=utf-8", contentType); + var element = ""; + + Assert.Equal(bytes, GetEnvelopedContent(element)); + } + + [Fact] + public void StoreUpdatesWhenExistsAndNewerExists() + { + byte[] bytes = null; + + var mock = new Mock(); + mock.Setup(c => c.GetPropertiesAsync( + It.IsAny(), + It.IsAny())) + .Returns(async (BlobRequestConditions conditions, CancellationToken token) => + { + var mockResponse = new Mock>(); + mockResponse.Setup(c => c.Value).Returns(new BlobProperties()); + + await Task.Yield(); + return mockResponse.Object; + }); + + mock.Setup(c => c.DownloadToAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(async (Stream target, BlobRequestConditions conditions, StorageTransferOptions options, CancellationToken token) => + { + var data = GetEnvelopedContent(""); + await target.WriteAsync(data, 0, data.Length); + + return new Mock().Object; + }) + .Verifiable(); + + mock.Setup(c => c.UploadAsync( + It.IsAny(), + It.IsAny(), + It.IsAny>(), + It.Is((BlobRequestConditions conditions) => conditions.IfNoneMatch == ETag.All), + It.IsAny>(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Throws(new RequestFailedException(status: 412, message: "")) + .Verifiable(); + + mock.Setup(c => c.UploadAsync( + It.IsAny(), + It.IsAny(), + It.IsAny>(), + It.Is((BlobRequestConditions conditions) => conditions.IfNoneMatch != ETag.All), + It.IsAny>(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(async (Stream strm, BlobHttpHeaders headers, IDictionary metaData, BlobRequestConditions conditions, IProgress progress, AccessTier? access, StorageTransferOptions transfer, CancellationToken token) => + { + using var memoryStream = new MemoryStream(); + strm.CopyTo(memoryStream); + bytes = memoryStream.ToArray(); + + await Task.Yield(); + + var mockResponse = new Mock>(); + var mockContentInfo = new Mock(); + mockContentInfo.Setup(c => c.ETag).Returns(ETag.All); + mockResponse.Setup(c => c.Value).Returns(mockContentInfo.Object); + return mockResponse.Object; + }) + .Verifiable(); + + var repository = new AzureBlobXmlRepository(() => mock.Object); + repository.StoreElement(new XElement("Element2"), null); + + mock.Verify(); + Assert.Equal(bytes, GetEnvelopedContent("")); + } + + private static byte[] GetEnvelopedContent(string element) + { + return Encoding.UTF8.GetBytes($"{element}"); + } + } +} diff --git a/src/DataProtection/Azure.Storage.Blob/test/AzureDataProtectionBuilderExtensionsTest.cs b/src/DataProtection/Azure.Storage.Blob/test/AzureDataProtectionBuilderExtensionsTest.cs new file mode 100644 index 000000000000..ef66f30247d0 --- /dev/null +++ b/src/DataProtection/Azure.Storage.Blob/test/AzureDataProtectionBuilderExtensionsTest.cs @@ -0,0 +1,32 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Azure.Storage.Blobs; +using Microsoft.AspNetCore.DataProtection.KeyManagement; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Xunit; + +namespace Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob +{ + public class AzureDataProtectionBuilderExtensionsTest + { + [Fact] + public void PersistKeysToAzureBlobStorage_UsesAzureBlobXmlRepository() + { + // Arrange + var container = new BlobContainerClient(new Uri("http://www.example.com")); + var serviceCollection = new ServiceCollection(); + var builder = serviceCollection.AddDataProtection(); + + // Act + builder.PersistKeysToAzureBlobStorage(container, "keys.xml"); + var services = serviceCollection.BuildServiceProvider(); + + // Assert + var options = services.GetRequiredService>(); + Assert.IsType(options.Value.XmlRepository); + } + } +} diff --git a/src/DataProtection/Azure.Storage.Blob/test/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.Tests.csproj b/src/DataProtection/Azure.Storage.Blob/test/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.Tests.csproj new file mode 100644 index 000000000000..389b5fc336b3 --- /dev/null +++ b/src/DataProtection/Azure.Storage.Blob/test/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.Tests.csproj @@ -0,0 +1,20 @@ + + + + $(DefaultNetCoreTargetFramework) + true + true + + false + + + + + + + + + + + + diff --git a/src/DataProtection/DataProtection.sln b/src/DataProtection/DataProtection.sln index 6726d0ab6d77..0271b73f1e69 100644 --- a/src/DataProtection/DataProtection.sln +++ b/src/DataProtection/DataProtection.sln @@ -73,6 +73,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.AspNetCore.DataPr EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EntityFrameworkCoreSample", "samples\EntityFrameworkCoreSample\EntityFrameworkCoreSample.csproj", "{DA4C8B07-05F5-4C59-A578-7438E9BFF79F}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Azure.Storage.Blob", "Azure.Storage.Blob", "{D2D67254-3071-46FE-8006-0E91DCBC0B9A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob", "Azure.Storage.Blob\src\Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj", "{8C348A99-541B-445A-824F-2CBD7550EEB2}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.Tests", "Azure.Storage.Blob\test\Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.Tests.csproj", "{FCAAE280-D132-4003-BD96-641F5962C425}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -383,6 +389,30 @@ Global {DA4C8B07-05F5-4C59-A578-7438E9BFF79F}.Release|x64.Build.0 = Release|Any CPU {DA4C8B07-05F5-4C59-A578-7438E9BFF79F}.Release|x86.ActiveCfg = Release|Any CPU {DA4C8B07-05F5-4C59-A578-7438E9BFF79F}.Release|x86.Build.0 = Release|Any CPU + {8C348A99-541B-445A-824F-2CBD7550EEB2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8C348A99-541B-445A-824F-2CBD7550EEB2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8C348A99-541B-445A-824F-2CBD7550EEB2}.Debug|x64.ActiveCfg = Debug|Any CPU + {8C348A99-541B-445A-824F-2CBD7550EEB2}.Debug|x64.Build.0 = Debug|Any CPU + {8C348A99-541B-445A-824F-2CBD7550EEB2}.Debug|x86.ActiveCfg = Debug|Any CPU + {8C348A99-541B-445A-824F-2CBD7550EEB2}.Debug|x86.Build.0 = Debug|Any CPU + {8C348A99-541B-445A-824F-2CBD7550EEB2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8C348A99-541B-445A-824F-2CBD7550EEB2}.Release|Any CPU.Build.0 = Release|Any CPU + {8C348A99-541B-445A-824F-2CBD7550EEB2}.Release|x64.ActiveCfg = Release|Any CPU + {8C348A99-541B-445A-824F-2CBD7550EEB2}.Release|x64.Build.0 = Release|Any CPU + {8C348A99-541B-445A-824F-2CBD7550EEB2}.Release|x86.ActiveCfg = Release|Any CPU + {8C348A99-541B-445A-824F-2CBD7550EEB2}.Release|x86.Build.0 = Release|Any CPU + {FCAAE280-D132-4003-BD96-641F5962C425}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FCAAE280-D132-4003-BD96-641F5962C425}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FCAAE280-D132-4003-BD96-641F5962C425}.Debug|x64.ActiveCfg = Debug|Any CPU + {FCAAE280-D132-4003-BD96-641F5962C425}.Debug|x64.Build.0 = Debug|Any CPU + {FCAAE280-D132-4003-BD96-641F5962C425}.Debug|x86.ActiveCfg = Debug|Any CPU + {FCAAE280-D132-4003-BD96-641F5962C425}.Debug|x86.Build.0 = Debug|Any CPU + {FCAAE280-D132-4003-BD96-641F5962C425}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FCAAE280-D132-4003-BD96-641F5962C425}.Release|Any CPU.Build.0 = Release|Any CPU + {FCAAE280-D132-4003-BD96-641F5962C425}.Release|x64.ActiveCfg = Release|Any CPU + {FCAAE280-D132-4003-BD96-641F5962C425}.Release|x64.Build.0 = Release|Any CPU + {FCAAE280-D132-4003-BD96-641F5962C425}.Release|x86.ActiveCfg = Release|Any CPU + {FCAAE280-D132-4003-BD96-641F5962C425}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -413,6 +443,8 @@ Global {8A7D0D2D-A5F1-4DF7-BBAA-9A0EFDBB5224} = {64FD02D7-B6F4-4C77-A3F8-E6BD6404168E} {74CE0E8B-DE23-4B53-8D02-69D6FB849ADC} = {64FD02D7-B6F4-4C77-A3F8-E6BD6404168E} {DA4C8B07-05F5-4C59-A578-7438E9BFF79F} = {9DF098B3-C8ED-471C-AE03-52E3196C1811} + {8C348A99-541B-445A-824F-2CBD7550EEB2} = {D2D67254-3071-46FE-8006-0E91DCBC0B9A} + {FCAAE280-D132-4003-BD96-641F5962C425} = {D2D67254-3071-46FE-8006-0E91DCBC0B9A} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {EA6767C5-D46B-4DE7-AB1A-E5244F122C64} From 45e18d50073a7d36c99ca1317d1a6c5f59cdf7c7 Mon Sep 17 00:00:00 2001 From: Brennan Conroy Date: Tue, 7 Jan 2020 16:29:02 -0800 Subject: [PATCH 02/15] progress --- .../Azure.Storage.Blob/src/AzureBlobXmlRepository.cs | 1 + .../test/AzureBlobXmlRepositoryTests.cs | 11 +++++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/DataProtection/Azure.Storage.Blob/src/AzureBlobXmlRepository.cs b/src/DataProtection/Azure.Storage.Blob/src/AzureBlobXmlRepository.cs index b401146450d1..f9c661efcf5b 100644 --- a/src/DataProtection/Azure.Storage.Blob/src/AzureBlobXmlRepository.cs +++ b/src/DataProtection/Azure.Storage.Blob/src/AzureBlobXmlRepository.cs @@ -222,6 +222,7 @@ private async Task StoreElementAsync(BlobClient blobRef, XElement element) var serializedDoc = new MemoryStream(); doc.Save(serializedDoc, SaveOptions.DisableFormatting); + serializedDoc.Position = 0; // Generate the appropriate precondition header based on whether or not // we believe data already exists in storage. diff --git a/src/DataProtection/Azure.Storage.Blob/test/AzureBlobXmlRepositoryTests.cs b/src/DataProtection/Azure.Storage.Blob/test/AzureBlobXmlRepositoryTests.cs index cabca9a9887b..e0c491a4e398 100644 --- a/src/DataProtection/Azure.Storage.Blob/test/AzureBlobXmlRepositoryTests.cs +++ b/src/DataProtection/Azure.Storage.Blob/test/AzureBlobXmlRepositoryTests.cs @@ -49,9 +49,9 @@ public void StoreCreatesBlobWhenNotExist() await Task.Yield(); var mockResponse = new Mock>(); - var mockContentInfo = new Mock(); - mockContentInfo.Setup(c => c.ETag).Returns(ETag.All); - mockResponse.Setup(c => c.Value).Returns(mockContentInfo.Object); + var blobContentInfo = BlobsModelFactory.BlobContentInfo(ETag.All, DateTimeOffset.Now.AddDays(-1), Array.Empty(), "", 1); + + mockResponse.Setup(c => c.Value).Returns(blobContentInfo); return mockResponse.Object; }); @@ -127,9 +127,8 @@ public void StoreUpdatesWhenExistsAndNewerExists() await Task.Yield(); var mockResponse = new Mock>(); - var mockContentInfo = new Mock(); - mockContentInfo.Setup(c => c.ETag).Returns(ETag.All); - mockResponse.Setup(c => c.Value).Returns(mockContentInfo.Object); + var blobContentInfo = BlobsModelFactory.BlobContentInfo(ETag.All, DateTimeOffset.Now.AddDays(-1), Array.Empty(), "", 1); + mockResponse.Setup(c => c.Value).Returns(blobContentInfo); return mockResponse.Object; }) .Verifiable(); From 21ad577865bc2671974f1cee2d4e41bc5d7fcccd Mon Sep 17 00:00:00 2001 From: Brennan Conroy Date: Wed, 8 Jan 2020 10:06:06 -0800 Subject: [PATCH 03/15] cleanup --- .../Azure.Storage.Blob/src/AzureBlobXmlRepository.cs | 4 ++-- .../Azure.Storage.Blob/test/AzureBlobXmlRepositoryTests.cs | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/DataProtection/Azure.Storage.Blob/src/AzureBlobXmlRepository.cs b/src/DataProtection/Azure.Storage.Blob/src/AzureBlobXmlRepository.cs index f9c661efcf5b..7cb005b20341 100644 --- a/src/DataProtection/Azure.Storage.Blob/src/AzureBlobXmlRepository.cs +++ b/src/DataProtection/Azure.Storage.Blob/src/AzureBlobXmlRepository.cs @@ -11,10 +11,10 @@ using System.Threading.Tasks; using System.Xml; using System.Xml.Linq; -using Microsoft.AspNetCore.DataProtection.Repositories; +using Azure; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; -using Azure; +using Microsoft.AspNetCore.DataProtection.Repositories; namespace Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob { diff --git a/src/DataProtection/Azure.Storage.Blob/test/AzureBlobXmlRepositoryTests.cs b/src/DataProtection/Azure.Storage.Blob/test/AzureBlobXmlRepositoryTests.cs index e0c491a4e398..b7fa15b01f4c 100644 --- a/src/DataProtection/Azure.Storage.Blob/test/AzureBlobXmlRepositoryTests.cs +++ b/src/DataProtection/Azure.Storage.Blob/test/AzureBlobXmlRepositoryTests.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; From 0c0ec46f4b5236d566842f8dda73582780072d7b Mon Sep 17 00:00:00 2001 From: Brennan Conroy Date: Wed, 8 Jan 2020 10:10:14 -0800 Subject: [PATCH 04/15] pre release --- .../Azure.Storage.Blob/Directory.Build.props | 10 ++++++++++ ...AspNetCore.DataProtection.Azure.Storage.Blob.csproj | 1 + 2 files changed, 11 insertions(+) create mode 100644 src/DataProtection/Azure.Storage.Blob/Directory.Build.props diff --git a/src/DataProtection/Azure.Storage.Blob/Directory.Build.props b/src/DataProtection/Azure.Storage.Blob/Directory.Build.props new file mode 100644 index 000000000000..6681da22e709 --- /dev/null +++ b/src/DataProtection/Azure.Storage.Blob/Directory.Build.props @@ -0,0 +1,10 @@ + + + + + + true + preview1 + + + \ No newline at end of file diff --git a/src/DataProtection/Azure.Storage.Blob/src/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj b/src/DataProtection/Azure.Storage.Blob/src/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj index 78a7b69ea7a2..e753a61f4e6f 100644 --- a/src/DataProtection/Azure.Storage.Blob/src/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj +++ b/src/DataProtection/Azure.Storage.Blob/src/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj @@ -8,6 +8,7 @@ aspnetcore;dataprotection;azure;blob true true + false From e0c54ffb394b55fe7d13c8101c128216f8e5636d Mon Sep 17 00:00:00 2001 From: Brennan Conroy Date: Wed, 8 Jan 2020 10:54:44 -0800 Subject: [PATCH 05/15] tried it out --- .../Azure.Storage.Blob/src/AzureBlobXmlRepository.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DataProtection/Azure.Storage.Blob/src/AzureBlobXmlRepository.cs b/src/DataProtection/Azure.Storage.Blob/src/AzureBlobXmlRepository.cs index 7cb005b20341..4997c530176f 100644 --- a/src/DataProtection/Azure.Storage.Blob/src/AzureBlobXmlRepository.cs +++ b/src/DataProtection/Azure.Storage.Blob/src/AzureBlobXmlRepository.cs @@ -109,7 +109,7 @@ private async Task> GetAllElementsAsync(BlobClient blobRef) { var data = await GetLatestDataAsync(blobRef); - if (data == null) + if (data == null || data.BlobContents.Length == 0) { // no data in blob storage return new XElement[0]; From fbea660b8d33a02fe50d2790bb5d0297bbe0d6c6 Mon Sep 17 00:00:00 2001 From: Brennan Conroy Date: Wed, 8 Jan 2020 12:39:33 -0800 Subject: [PATCH 06/15] code check --- eng/ProjectReferences.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/ProjectReferences.props b/eng/ProjectReferences.props index c1a4cf7ff36b..74d9a22da1a6 100644 --- a/eng/ProjectReferences.props +++ b/eng/ProjectReferences.props @@ -6,6 +6,7 @@ + @@ -63,7 +64,6 @@ - From 6fb928fc0650f87474e2a4829102472646dabe6f Mon Sep 17 00:00:00 2001 From: Brennan Conroy Date: Thu, 9 Jan 2020 09:42:20 -0800 Subject: [PATCH 07/15] cleanup --- eng/Versions.props | 1 + .../Azure.Storage.Blob/Directory.Build.props | 10 ---------- ...AspNetCore.DataProtection.Azure.Storage.Blob.csproj | 2 ++ 3 files changed, 3 insertions(+), 10 deletions(-) delete mode 100644 src/DataProtection/Azure.Storage.Blob/Directory.Build.props diff --git a/eng/Versions.props b/eng/Versions.props index b7ac67bccb7b..1ae80e2456ca 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -22,6 +22,7 @@ 4 preview$(BlazorClientPreReleasePreviewNumber) + preview1 $(AspNetCoreMajorVersion).$(AspNetCoreMinorVersion) 3.1.0 diff --git a/src/DataProtection/Azure.Storage.Blob/Directory.Build.props b/src/DataProtection/Azure.Storage.Blob/Directory.Build.props deleted file mode 100644 index 6681da22e709..000000000000 --- a/src/DataProtection/Azure.Storage.Blob/Directory.Build.props +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - true - preview1 - - - \ No newline at end of file diff --git a/src/DataProtection/Azure.Storage.Blob/src/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj b/src/DataProtection/Azure.Storage.Blob/src/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj index e753a61f4e6f..4823744c9db0 100644 --- a/src/DataProtection/Azure.Storage.Blob/src/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj +++ b/src/DataProtection/Azure.Storage.Blob/src/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj @@ -9,6 +9,8 @@ true true false + $(AzureSDKPreReleaseVersionLabel) + From 7494632325d0968dd8d7cae852d90b4c9b5e26c2 Mon Sep 17 00:00:00 2001 From: BrennanConroy Date: Mon, 13 Jan 2020 13:33:09 -0800 Subject: [PATCH 08/15] fb --- eng/Dependencies.props | 1 + eng/Versions.props | 1 + .../src/AzureBlobXmlRepository.cs | 58 ++++-------- .../AzureDataProtectionBuilderExtensions.cs | 77 +++++++++++----- ...e.DataProtection.Azure.Storage.Blob.csproj | 4 +- .../test/AzureBlobXmlRepositoryTests.cs | 20 ++--- .../Azure.Storage.Blob/test/MockResponse.cs | 90 +++++++++++++++++++ 7 files changed, 170 insertions(+), 81 deletions(-) create mode 100644 src/DataProtection/Azure.Storage.Blob/test/MockResponse.cs diff --git a/eng/Dependencies.props b/eng/Dependencies.props index 647043d7c413..cf927f94a648 100644 --- a/eng/Dependencies.props +++ b/eng/Dependencies.props @@ -164,6 +164,7 @@ and are generated based on the last package release. + diff --git a/eng/Versions.props b/eng/Versions.props index 1ae80e2456ca..ac1b6586755e 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -235,6 +235,7 @@ 0.9.9 12.1.0 + 1.0.0 0.10.13 4.2.1 4.2.1 diff --git a/src/DataProtection/Azure.Storage.Blob/src/AzureBlobXmlRepository.cs b/src/DataProtection/Azure.Storage.Blob/src/AzureBlobXmlRepository.cs index 4997c530176f..6e766a18a6d9 100644 --- a/src/DataProtection/Azure.Storage.Blob/src/AzureBlobXmlRepository.cs +++ b/src/DataProtection/Azure.Storage.Blob/src/AzureBlobXmlRepository.cs @@ -31,31 +31,27 @@ public sealed class AzureBlobXmlRepository : IXmlRepository private static readonly XName RepositoryElementName = "repository"; - private readonly Func _blobRefFactory; private readonly Random _random; private BlobData _cachedBlobData; + private readonly BlobClient _blobClient; /// /// Creates a new instance of the . /// - /// A factory which can create - /// instances. The factory must be thread-safe for invocation by multiple - /// concurrent threads, and each invocation must return a new object. - public AzureBlobXmlRepository(Func blobRefFactory) + /// A that is connected to the blob we are reading from and writing to. + public AzureBlobXmlRepository(BlobClient blobClient) { - _blobRefFactory = blobRefFactory ?? throw new ArgumentNullException(nameof(blobRefFactory)); _random = new Random(); + _blobClient = blobClient; } /// public IReadOnlyCollection GetAllElements() { - var blobRef = CreateFreshBlobRef(); - // Shunt the work onto a ThreadPool thread so that it's independent of any // existing sync context or other potentially deadlock-causing items. - var elements = Task.Run(() => GetAllElementsAsync(blobRef)).GetAwaiter().GetResult(); + var elements = Task.Run(() => GetAllElementsAsync()).GetAwaiter().GetResult(); return new ReadOnlyCollection(elements); } @@ -67,12 +63,10 @@ public void StoreElement(XElement element, string friendlyName) throw new ArgumentNullException(nameof(element)); } - var blobRef = CreateFreshBlobRef(); - // Shunt the work onto a ThreadPool thread so that it's independent of any // existing sync context or other potentially deadlock-causing items. - Task.Run(() => StoreElementAsync(blobRef, element)).GetAwaiter().GetResult(); + Task.Run(() => StoreElementAsync(element)).GetAwaiter().GetResult(); } private XDocument CreateDocumentFromBlob(byte[] blob) @@ -91,23 +85,9 @@ private XDocument CreateDocumentFromBlob(byte[] blob) } } - private BlobClient CreateFreshBlobRef() + private async Task> GetAllElementsAsync() { - // ICloudBlob instances aren't thread-safe, so we need to make sure we're working - // with a fresh instance that won't be mutated by another thread. - - var blobRef = _blobRefFactory(); - if (blobRef == null) - { - throw new InvalidOperationException("The ICloudBlob factory method returned null."); - } - - return blobRef; - } - - private async Task> GetAllElementsAsync(BlobClient blobRef) - { - var data = await GetLatestDataAsync(blobRef); + var data = await GetLatestDataAsync(); if (data == null || data.BlobContents.Length == 0) { @@ -129,7 +109,7 @@ private async Task> GetAllElementsAsync(BlobClient blobRef) return doc.Root.Elements().ToList(); } - private async Task GetLatestDataAsync(BlobClient blobRef) + private async Task GetLatestDataAsync() { // Set the appropriate AccessCondition based on what we believe the latest // file contents to be, then make the request. @@ -143,19 +123,16 @@ private async Task GetLatestDataAsync(BlobClient blobRef) { using (var memoryStream = new MemoryStream()) { - await blobRef.DownloadToAsync( + var response = await _blobClient.DownloadToAsync( destination: memoryStream, conditions: requestCondition); // At this point, our original cache either didn't exist or was outdated. - // We'll update it now and return the updated value; - - var props = await blobRef.GetPropertiesAsync(); - + // We'll update it now and return the updated value latestCachedData = new BlobData() { BlobContents = memoryStream.ToArray(), - ETag = props.Value.ETag + ETag = response.Headers.ETag }; } @@ -188,7 +165,7 @@ private int GetRandomizedBackoffPeriod() return (int) (multiplier * ConflictBackoffPeriod.Ticks); } - private async Task StoreElementAsync(BlobClient blobRef, XElement element) + private async Task StoreElementAsync(XElement element) { // holds the last error in case we need to rethrow it ExceptionDispatchInfo lastError = null; @@ -206,7 +183,7 @@ private async Task StoreElementAsync(BlobClient blobRef, XElement element) { // If at least one conflict occurred, make sure we have an up-to-date // view of the blob contents. - await GetLatestDataAsync(blobRef); + await GetLatestDataAsync(); } // Merge the new element into the document. If no document exists, @@ -231,8 +208,7 @@ private async Task StoreElementAsync(BlobClient blobRef, XElement element) BlobHttpHeaders headers = null; if (latestData != null) { - var props = await blobRef.GetPropertiesAsync(); - requestConditions = new BlobRequestConditions() { IfMatch = props.Value.ETag }; + requestConditions = new BlobRequestConditions() { IfMatch = latestData.ETag }; } else { @@ -244,7 +220,7 @@ private async Task StoreElementAsync(BlobClient blobRef, XElement element) try { // Send the request up to the server. - var response = await blobRef.UploadAsync( + var response = await _blobClient.UploadAsync( serializedDoc, httpHeaders: headers, conditions: requestConditions); @@ -285,7 +261,7 @@ private async Task StoreElementAsync(BlobClient blobRef, XElement element) private sealed class BlobData { internal byte[] BlobContents; - internal ETag ETag; + internal ETag? ETag; } } } diff --git a/src/DataProtection/Azure.Storage.Blob/src/AzureDataProtectionBuilderExtensions.cs b/src/DataProtection/Azure.Storage.Blob/src/AzureDataProtectionBuilderExtensions.cs index 4e116112792b..60768a136bab 100644 --- a/src/DataProtection/Azure.Storage.Blob/src/AzureDataProtectionBuilderExtensions.cs +++ b/src/DataProtection/Azure.Storage.Blob/src/AzureDataProtectionBuilderExtensions.cs @@ -2,6 +2,8 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using Azure.Core; +using Azure.Identity; using Azure.Storage; using Azure.Storage.Blobs; using Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob; @@ -39,24 +41,57 @@ public static IDataProtectionBuilder PersistKeysToAzureBlobStorage(this IDataPro } var uriBuilder = new BlobUriBuilder(blobUri); + var blobName = uriBuilder.BlobName; + BlobContainerClient client; // The SAS token is present in the query string. + if (uriBuilder.Sas == null) + { + client = new BlobContainerClient(blobUri, new DefaultAzureCredential()); + } + else + { + var credentials = new StorageSharedKeyCredential(uriBuilder.AccountName, uriBuilder.Query); + uriBuilder.Query = null; // no longer needed + var blobAbsoluteUri = uriBuilder.ToUri(); + client = new BlobContainerClient(blobAbsoluteUri, credentials); + } + + return PersistKeystoAzureBlobStorageInternal(builder, client.GetBlobClient(blobName)); + } - if (string.IsNullOrEmpty(uriBuilder.Query)) + /// + /// Configures the data protection system to persist keys to the specified path + /// in Azure Blob Storage. + /// + /// The builder instance to modify. + /// The full URI where the key file should be stored. + /// The URI must contain the SAS token as a query string parameter. + /// The credentials to connect to the blob. + /// The value . + /// + /// The container referenced by must already exist. + /// + public static IDataProtectionBuilder PersistKeysToAzureBlobStorage(this IDataProtectionBuilder builder, Uri blobUri, TokenCredential tokenCredential) + { + if (builder == null) { - throw new ArgumentException( - message: "URI does not have a SAS token in the query string.", - paramName: nameof(blobUri)); + throw new ArgumentNullException(nameof(builder)); + } + if (blobUri == null) + { + throw new ArgumentNullException(nameof(blobUri)); + } + if (tokenCredential == null) + { + throw new ArgumentNullException(nameof(tokenCredential)); } - var credentials = new StorageSharedKeyCredential(uriBuilder.AccountName, uriBuilder.Query); + var client = new BlobContainerClient(blobUri, tokenCredential); + var uriBuilder = new BlobUriBuilder(blobUri); var blobName = uriBuilder.BlobName; - uriBuilder.Query = null; // no longer needed - var blobAbsoluteUri = uriBuilder.ToUri(); - var client = new BlobContainerClient(blobAbsoluteUri, credentials); - - return PersistKeystoAzureBlobStorageInternal(builder, () => client.GetBlobClient(blobName)); + return PersistKeystoAzureBlobStorageInternal(builder, client.GetBlobClient(blobName)); } /// @@ -64,37 +99,31 @@ public static IDataProtectionBuilder PersistKeysToAzureBlobStorage(this IDataPro /// in Azure Blob Storage. /// /// The builder instance to modify. - /// The in which the + /// The in which the /// key file should be stored. - /// The name of the key file, generally specified - /// as "[subdir/]keys.xml" /// The value . /// - /// The container referenced by must already exist. + /// The blob referenced by must already exist. /// - public static IDataProtectionBuilder PersistKeysToAzureBlobStorage(this IDataProtectionBuilder builder, BlobContainerClient container, string blobName) + public static IDataProtectionBuilder PersistKeysToAzureBlobStorage(this IDataProtectionBuilder builder, BlobClient blobClient) { if (builder == null) { throw new ArgumentNullException(nameof(builder)); } - if (container == null) - { - throw new ArgumentNullException(nameof(container)); - } - if (blobName == null) + if (blobClient == null) { - throw new ArgumentNullException(nameof(blobName)); + throw new ArgumentNullException(nameof(blobClient)); } - return PersistKeystoAzureBlobStorageInternal(builder, () => container.GetBlobClient(blobName)); + return PersistKeystoAzureBlobStorageInternal(builder, blobClient); } // important: the Func passed into this method must return a new instance with each call - private static IDataProtectionBuilder PersistKeystoAzureBlobStorageInternal(IDataProtectionBuilder builder, Func blobRefFactory) + private static IDataProtectionBuilder PersistKeystoAzureBlobStorageInternal(IDataProtectionBuilder builder, BlobClient blobClient) { builder.Services.Configure(options => { - options.XmlRepository = new AzureBlobXmlRepository(blobRefFactory); + options.XmlRepository = new AzureBlobXmlRepository(blobClient); }); return builder; } diff --git a/src/DataProtection/Azure.Storage.Blob/src/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj b/src/DataProtection/Azure.Storage.Blob/src/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj index 4823744c9db0..b6f03510223d 100644 --- a/src/DataProtection/Azure.Storage.Blob/src/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj +++ b/src/DataProtection/Azure.Storage.Blob/src/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj @@ -1,4 +1,4 @@ - + Microsoft Azure Blob storage support as key store. @@ -8,7 +8,6 @@ aspnetcore;dataprotection;azure;blob true true - false $(AzureSDKPreReleaseVersionLabel) @@ -16,6 +15,7 @@ + diff --git a/src/DataProtection/Azure.Storage.Blob/test/AzureBlobXmlRepositoryTests.cs b/src/DataProtection/Azure.Storage.Blob/test/AzureBlobXmlRepositoryTests.cs index b7fa15b01f4c..8d6c111a32cf 100644 --- a/src/DataProtection/Azure.Storage.Blob/test/AzureBlobXmlRepositoryTests.cs +++ b/src/DataProtection/Azure.Storage.Blob/test/AzureBlobXmlRepositoryTests.cs @@ -9,6 +9,7 @@ using System.Threading.Tasks; using System.Xml.Linq; using Azure; +using Azure.Core; using Azure.Storage; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; @@ -54,7 +55,7 @@ public void StoreCreatesBlobWhenNotExist() return mockResponse.Object; }); - var repository = new AzureBlobXmlRepository(() => mock.Object); + var repository = new AzureBlobXmlRepository(mock.Object); repository.StoreElement(new XElement("Element"), null); Assert.Equal("*", uploadConditions.IfNoneMatch.ToString()); @@ -70,17 +71,6 @@ public void StoreUpdatesWhenExistsAndNewerExists() byte[] bytes = null; var mock = new Mock(); - mock.Setup(c => c.GetPropertiesAsync( - It.IsAny(), - It.IsAny())) - .Returns(async (BlobRequestConditions conditions, CancellationToken token) => - { - var mockResponse = new Mock>(); - mockResponse.Setup(c => c.Value).Returns(new BlobProperties()); - - await Task.Yield(); - return mockResponse.Object; - }); mock.Setup(c => c.DownloadToAsync( It.IsAny(), @@ -92,7 +82,9 @@ public void StoreUpdatesWhenExistsAndNewerExists() var data = GetEnvelopedContent(""); await target.WriteAsync(data, 0, data.Length); - return new Mock().Object; + var response = new MockResponse(200); + response.AddHeader(new HttpHeader("ETag", "*")); + return response; }) .Verifiable(); @@ -132,7 +124,7 @@ public void StoreUpdatesWhenExistsAndNewerExists() }) .Verifiable(); - var repository = new AzureBlobXmlRepository(() => mock.Object); + var repository = new AzureBlobXmlRepository(mock.Object); repository.StoreElement(new XElement("Element2"), null); mock.Verify(); diff --git a/src/DataProtection/Azure.Storage.Blob/test/MockResponse.cs b/src/DataProtection/Azure.Storage.Blob/test/MockResponse.cs new file mode 100644 index 000000000000..f1901ddef4d7 --- /dev/null +++ b/src/DataProtection/Azure.Storage.Blob/test/MockResponse.cs @@ -0,0 +1,90 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using Azure; +using Azure.Core; + +namespace Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.Test +{ + public class MockResponse : Response + { + private readonly Dictionary> _headers = new Dictionary>(StringComparer.OrdinalIgnoreCase); + + public MockResponse(int status, string reasonPhrase = null) + { + Status = status; + ReasonPhrase = reasonPhrase; + } + + public override int Status { get; } + + public override string ReasonPhrase { get; } + + public override Stream ContentStream { get; set; } + + public override string ClientRequestId { get; set; } + + public bool IsDisposed { get; private set; } + + public void SetContent(byte[] content) + { + ContentStream = new MemoryStream(content); + } + + public void SetContent(string content) + { + SetContent(Encoding.UTF8.GetBytes(content)); + } + + public void AddHeader(HttpHeader header) + { + if (!_headers.TryGetValue(header.Name, out List values)) + { + _headers[header.Name] = values = new List(); + } + + values.Add(header.Value); + } + + protected override bool TryGetHeader(string name, out string value) + { + if (_headers.TryGetValue(name, out List values)) + { + value = JoinHeaderValue(values); + return true; + } + + value = null; + return false; + } + + protected override bool TryGetHeaderValues(string name, out IEnumerable values) + { + var result = _headers.TryGetValue(name, out List valuesList); + values = valuesList; + return result; + } + + protected override bool ContainsHeader(string name) + { + return TryGetHeaderValues(name, out _); + } + + protected override IEnumerable EnumerateHeaders() => _headers.Select(h => new HttpHeader(h.Key, JoinHeaderValue(h.Value))); + + private static string JoinHeaderValue(IEnumerable values) + { + return string.Join(",", values); + } + + public override void Dispose() + { + IsDisposed = true; + } + } +} From 2792c41c10aca43fd28f2038219454d98b524b49 Mon Sep 17 00:00:00 2001 From: BrennanConroy Date: Mon, 13 Jan 2020 14:13:53 -0800 Subject: [PATCH 09/15] progress --- .../AzureDataProtectionBuilderExtensions.cs | 20 ++++++------------- ...NetCore.DataProtection.AzureStorage.csproj | 6 ------ 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/src/DataProtection/Azure.Storage.Blob/src/AzureDataProtectionBuilderExtensions.cs b/src/DataProtection/Azure.Storage.Blob/src/AzureDataProtectionBuilderExtensions.cs index 60768a136bab..c267d9d35c85 100644 --- a/src/DataProtection/Azure.Storage.Blob/src/AzureDataProtectionBuilderExtensions.cs +++ b/src/DataProtection/Azure.Storage.Blob/src/AzureDataProtectionBuilderExtensions.cs @@ -4,7 +4,6 @@ using System; using Azure.Core; using Azure.Identity; -using Azure.Storage; using Azure.Storage.Blobs; using Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob; using Microsoft.AspNetCore.DataProtection.KeyManagement; @@ -41,23 +40,19 @@ public static IDataProtectionBuilder PersistKeysToAzureBlobStorage(this IDataPro } var uriBuilder = new BlobUriBuilder(blobUri); - var blobName = uriBuilder.BlobName; - BlobContainerClient client; + BlobClient client; // The SAS token is present in the query string. if (uriBuilder.Sas == null) { - client = new BlobContainerClient(blobUri, new DefaultAzureCredential()); + client = new BlobClient(blobUri, new DefaultAzureCredential()); } else { - var credentials = new StorageSharedKeyCredential(uriBuilder.AccountName, uriBuilder.Query); - uriBuilder.Query = null; // no longer needed - var blobAbsoluteUri = uriBuilder.ToUri(); - client = new BlobContainerClient(blobAbsoluteUri, credentials); + client = new BlobClient(blobUri); } - return PersistKeystoAzureBlobStorageInternal(builder, client.GetBlobClient(blobName)); + return PersistKeystoAzureBlobStorageInternal(builder, client); } /// @@ -87,11 +82,9 @@ public static IDataProtectionBuilder PersistKeysToAzureBlobStorage(this IDataPro throw new ArgumentNullException(nameof(tokenCredential)); } - var client = new BlobContainerClient(blobUri, tokenCredential); - var uriBuilder = new BlobUriBuilder(blobUri); - var blobName = uriBuilder.BlobName; + var client = new BlobClient(blobUri, tokenCredential); - return PersistKeystoAzureBlobStorageInternal(builder, client.GetBlobClient(blobName)); + return PersistKeystoAzureBlobStorageInternal(builder, client); } /// @@ -118,7 +111,6 @@ public static IDataProtectionBuilder PersistKeysToAzureBlobStorage(this IDataPro return PersistKeystoAzureBlobStorageInternal(builder, blobClient); } - // important: the Func passed into this method must return a new instance with each call private static IDataProtectionBuilder PersistKeystoAzureBlobStorageInternal(IDataProtectionBuilder builder, BlobClient blobClient) { builder.Services.Configure(options => diff --git a/src/DataProtection/AzureStorage/src/Microsoft.AspNetCore.DataProtection.AzureStorage.csproj b/src/DataProtection/AzureStorage/src/Microsoft.AspNetCore.DataProtection.AzureStorage.csproj index b2e913f35262..932a82ec15d0 100644 --- a/src/DataProtection/AzureStorage/src/Microsoft.AspNetCore.DataProtection.AzureStorage.csproj +++ b/src/DataProtection/AzureStorage/src/Microsoft.AspNetCore.DataProtection.AzureStorage.csproj @@ -15,10 +15,4 @@ - - - - - - From 784f956457ee49a3947a4e40eb4f702e78435f3a Mon Sep 17 00:00:00 2001 From: BrennanConroy Date: Mon, 13 Jan 2020 14:15:11 -0800 Subject: [PATCH 10/15] whoops --- ...soft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj | 6 +----- .../Microsoft.AspNetCore.DataProtection.AzureStorage.csproj | 6 ++++++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/DataProtection/Azure.Storage.Blob/src/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj b/src/DataProtection/Azure.Storage.Blob/src/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj index b6f03510223d..f2f2e5a6b10c 100644 --- a/src/DataProtection/Azure.Storage.Blob/src/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj +++ b/src/DataProtection/Azure.Storage.Blob/src/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj @@ -1,4 +1,4 @@ - + Microsoft Azure Blob storage support as key store. @@ -18,8 +18,4 @@ - - - - diff --git a/src/DataProtection/AzureStorage/src/Microsoft.AspNetCore.DataProtection.AzureStorage.csproj b/src/DataProtection/AzureStorage/src/Microsoft.AspNetCore.DataProtection.AzureStorage.csproj index 932a82ec15d0..b2e913f35262 100644 --- a/src/DataProtection/AzureStorage/src/Microsoft.AspNetCore.DataProtection.AzureStorage.csproj +++ b/src/DataProtection/AzureStorage/src/Microsoft.AspNetCore.DataProtection.AzureStorage.csproj @@ -15,4 +15,10 @@ + + + + + + From 48cb40f33e2603c8d22d3ad423e7dbdda6869c9c Mon Sep 17 00:00:00 2001 From: BrennanConroy Date: Mon, 13 Jan 2020 14:54:08 -0800 Subject: [PATCH 11/15] test --- .../test/AzureDataProtectionBuilderExtensionsTest.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/DataProtection/Azure.Storage.Blob/test/AzureDataProtectionBuilderExtensionsTest.cs b/src/DataProtection/Azure.Storage.Blob/test/AzureDataProtectionBuilderExtensionsTest.cs index ef66f30247d0..89b2f3a0aa69 100644 --- a/src/DataProtection/Azure.Storage.Blob/test/AzureDataProtectionBuilderExtensionsTest.cs +++ b/src/DataProtection/Azure.Storage.Blob/test/AzureDataProtectionBuilderExtensionsTest.cs @@ -16,12 +16,12 @@ public class AzureDataProtectionBuilderExtensionsTest public void PersistKeysToAzureBlobStorage_UsesAzureBlobXmlRepository() { // Arrange - var container = new BlobContainerClient(new Uri("http://www.example.com")); + var client = new BlobClient(new Uri("http://www.example.com")); var serviceCollection = new ServiceCollection(); var builder = serviceCollection.AddDataProtection(); // Act - builder.PersistKeysToAzureBlobStorage(container, "keys.xml"); + builder.PersistKeysToAzureBlobStorage(client); var services = serviceCollection.BuildServiceProvider(); // Assert From e22a8eba4f2f3af961c41e23c2452d1219850a04 Mon Sep 17 00:00:00 2001 From: BrennanConroy Date: Mon, 13 Jan 2020 15:36:07 -0800 Subject: [PATCH 12/15] fb --- THIRD-PARTY-NOTICES.txt | 24 +++++++++++++++++++ .../AzureDataProtectionBuilderExtensions.cs | 2 +- .../test/AzureBlobXmlRepositoryTests.cs | 2 +- .../Azure.Storage.Blob/test/MockResponse.cs | 3 ++- 4 files changed, 28 insertions(+), 3 deletions(-) diff --git a/THIRD-PARTY-NOTICES.txt b/THIRD-PARTY-NOTICES.txt index 81fadeae227f..1d8672060b5f 100644 --- a/THIRD-PARTY-NOTICES.txt +++ b/THIRD-PARTY-NOTICES.txt @@ -189,3 +189,27 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +License notice for azure-sdk-for-net +------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2015 Microsoft + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/src/DataProtection/Azure.Storage.Blob/src/AzureDataProtectionBuilderExtensions.cs b/src/DataProtection/Azure.Storage.Blob/src/AzureDataProtectionBuilderExtensions.cs index c267d9d35c85..f20a08208bc4 100644 --- a/src/DataProtection/Azure.Storage.Blob/src/AzureDataProtectionBuilderExtensions.cs +++ b/src/DataProtection/Azure.Storage.Blob/src/AzureDataProtectionBuilderExtensions.cs @@ -15,7 +15,7 @@ namespace Microsoft.AspNetCore.DataProtection /// Contains Azure-specific extension methods for modifying a /// . /// - public static class AzureDataProtectionBuilderExtensions + public static class AzureStorageBlobDataProtectionBuilderExtensions { /// /// Configures the data protection system to persist keys to the specified path diff --git a/src/DataProtection/Azure.Storage.Blob/test/AzureBlobXmlRepositoryTests.cs b/src/DataProtection/Azure.Storage.Blob/test/AzureBlobXmlRepositoryTests.cs index 8d6c111a32cf..7b46bc4a4010 100644 --- a/src/DataProtection/Azure.Storage.Blob/test/AzureBlobXmlRepositoryTests.cs +++ b/src/DataProtection/Azure.Storage.Blob/test/AzureBlobXmlRepositoryTests.cs @@ -16,7 +16,7 @@ using Moq; using Xunit; -namespace Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.Test +namespace Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob { public class AzureBlobXmlRepositoryTests { diff --git a/src/DataProtection/Azure.Storage.Blob/test/MockResponse.cs b/src/DataProtection/Azure.Storage.Blob/test/MockResponse.cs index f1901ddef4d7..735d2ea246ec 100644 --- a/src/DataProtection/Azure.Storage.Blob/test/MockResponse.cs +++ b/src/DataProtection/Azure.Storage.Blob/test/MockResponse.cs @@ -9,8 +9,9 @@ using Azure; using Azure.Core; -namespace Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.Test +namespace Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob { + // Copied from https://github.com/Azure/azure-sdk-for-net/blob/43a178cbb60a50d9841b48170681b9187143e551/sdk/core/Azure.Core/tests/TestFramework/MockResponse.cs public class MockResponse : Response { private readonly Dictionary> _headers = new Dictionary>(StringComparer.OrdinalIgnoreCase); From f96f63b968639d8c9f1cb79afcf28b80a7ced43b Mon Sep 17 00:00:00 2001 From: BrennanConroy Date: Mon, 13 Jan 2020 16:05:48 -0800 Subject: [PATCH 13/15] file name --- ...ions.cs => AzureStorageBlobDataProtectionBuilderExtensions.cs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/DataProtection/Azure.Storage.Blob/src/{AzureDataProtectionBuilderExtensions.cs => AzureStorageBlobDataProtectionBuilderExtensions.cs} (100%) diff --git a/src/DataProtection/Azure.Storage.Blob/src/AzureDataProtectionBuilderExtensions.cs b/src/DataProtection/Azure.Storage.Blob/src/AzureStorageBlobDataProtectionBuilderExtensions.cs similarity index 100% rename from src/DataProtection/Azure.Storage.Blob/src/AzureDataProtectionBuilderExtensions.cs rename to src/DataProtection/Azure.Storage.Blob/src/AzureStorageBlobDataProtectionBuilderExtensions.cs From f2cdde34a5cc31d8ec8d0f59ea370261ec7e6e01 Mon Sep 17 00:00:00 2001 From: BrennanConroy Date: Tue, 14 Jan 2020 09:32:09 -0800 Subject: [PATCH 14/15] version --- ...crosoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/DataProtection/Azure.Storage.Blob/src/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj b/src/DataProtection/Azure.Storage.Blob/src/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj index f2f2e5a6b10c..dbf1837f6715 100644 --- a/src/DataProtection/Azure.Storage.Blob/src/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj +++ b/src/DataProtection/Azure.Storage.Blob/src/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj @@ -1,4 +1,4 @@ - + Microsoft Azure Blob storage support as key store. @@ -10,6 +10,7 @@ true $(AzureSDKPreReleaseVersionLabel) + $(AspNetCoreMajorVersion).$(AspNetCoreMinorVersion).0 From 764edf3d75060dc8beb114d4774cf6e7849fc485 Mon Sep 17 00:00:00 2001 From: Brennan Date: Wed, 22 Jan 2020 10:48:06 -0800 Subject: [PATCH 15/15] Update src/DataProtection/Azure.Storage.Blob/src/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj --- ...Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/DataProtection/Azure.Storage.Blob/src/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj b/src/DataProtection/Azure.Storage.Blob/src/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj index dbf1837f6715..a37536c289b0 100644 --- a/src/DataProtection/Azure.Storage.Blob/src/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj +++ b/src/DataProtection/Azure.Storage.Blob/src/Microsoft.AspNetCore.DataProtection.Azure.Storage.Blob.csproj @@ -10,7 +10,6 @@ true $(AzureSDKPreReleaseVersionLabel) - $(AspNetCoreMajorVersion).$(AspNetCoreMinorVersion).0