diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/api/Azure.Storage.DataMovement.Blobs.net6.0.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/api/Azure.Storage.DataMovement.Blobs.net6.0.cs index 2326f9d10ffd..71ae2c4706fd 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/api/Azure.Storage.DataMovement.Blobs.net6.0.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/api/Azure.Storage.DataMovement.Blobs.net6.0.cs @@ -31,7 +31,7 @@ public BlobsStorageResourceProvider(Azure.Storage.DataMovement.Blobs.BlobsStorag public BlobsStorageResourceProvider(Azure.Storage.DataMovement.Blobs.BlobsStorageResourceProvider.GetStorageSharedKeyCredential getStorageSharedKeyCredentialAsync) { } public BlobsStorageResourceProvider(Azure.Storage.DataMovement.Blobs.BlobsStorageResourceProvider.GetTokenCredential getTokenCredentialAsync) { } public BlobsStorageResourceProvider(Azure.Storage.StorageSharedKeyCredential credential) { } - protected override string TypeId { get { throw null; } } + protected override string ProviderId { get { throw null; } } public Azure.Storage.DataMovement.StorageResource FromBlob(string blobUri, Azure.Storage.DataMovement.Blobs.BlobStorageResourceOptions options = null) { throw null; } public Azure.Storage.DataMovement.StorageResource FromClient(Azure.Storage.Blobs.BlobContainerClient client, Azure.Storage.DataMovement.Blobs.BlobStorageResourceContainerOptions options = null) { throw null; } public Azure.Storage.DataMovement.StorageResource FromClient(Azure.Storage.Blobs.Specialized.AppendBlobClient client, Azure.Storage.DataMovement.Blobs.AppendBlobStorageResourceOptions options = null) { throw null; } diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/api/Azure.Storage.DataMovement.Blobs.netstandard2.0.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/api/Azure.Storage.DataMovement.Blobs.netstandard2.0.cs index 2326f9d10ffd..71ae2c4706fd 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/api/Azure.Storage.DataMovement.Blobs.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/api/Azure.Storage.DataMovement.Blobs.netstandard2.0.cs @@ -31,7 +31,7 @@ public BlobsStorageResourceProvider(Azure.Storage.DataMovement.Blobs.BlobsStorag public BlobsStorageResourceProvider(Azure.Storage.DataMovement.Blobs.BlobsStorageResourceProvider.GetStorageSharedKeyCredential getStorageSharedKeyCredentialAsync) { } public BlobsStorageResourceProvider(Azure.Storage.DataMovement.Blobs.BlobsStorageResourceProvider.GetTokenCredential getTokenCredentialAsync) { } public BlobsStorageResourceProvider(Azure.Storage.StorageSharedKeyCredential credential) { } - protected override string TypeId { get { throw null; } } + protected override string ProviderId { get { throw null; } } public Azure.Storage.DataMovement.StorageResource FromBlob(string blobUri, Azure.Storage.DataMovement.Blobs.BlobStorageResourceOptions options = null) { throw null; } public Azure.Storage.DataMovement.StorageResource FromClient(Azure.Storage.Blobs.BlobContainerClient client, Azure.Storage.DataMovement.Blobs.BlobStorageResourceContainerOptions options = null) { throw null; } public Azure.Storage.DataMovement.StorageResource FromClient(Azure.Storage.Blobs.Specialized.AppendBlobClient client, Azure.Storage.DataMovement.Blobs.AppendBlobStorageResourceOptions options = null) { throw null; } diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/AppendBlobStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/AppendBlobStorageResource.cs index 6eaf526aaf43..80c85f3ffa61 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/AppendBlobStorageResource.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/AppendBlobStorageResource.cs @@ -20,16 +20,12 @@ internal class AppendBlobStorageResource : StorageResourceItemInternal internal long? _length; internal ETag? _etagDownloadLock = default; - /// - /// The identifier for the type of storage resource. - /// protected override string ResourceId => "AppendBlob"; - /// - /// Gets the Uri of the Storage Resource - /// public override Uri Uri => BlobClient.Uri; + public override string ProviderId => "blob"; + /// /// Defines the recommended Transfer Type for the storage resource. /// diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs index 5d12996b35e4..e07d9bfdce03 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs @@ -29,6 +29,8 @@ internal class BlobStorageResourceContainer : StorageResourceContainerInternal /// public override Uri Uri => _uri; + public override string ProviderId => "blob"; + /// /// The constructor to create an instance of the BlobStorageResourceContainer. /// diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobsStorageResourceProvider.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobsStorageResourceProvider.cs index 05956dcf7426..b658f9042632 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobsStorageResourceProvider.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobsStorageResourceProvider.cs @@ -66,7 +66,7 @@ private enum CredentialType } /// - protected override string TypeId => "blob"; + protected override string ProviderId => "blob"; private readonly CredentialType _credentialType; private readonly GetStorageSharedKeyCredential _getStorageSharedKeyCredential; diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlockBlobStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlockBlobStorageResource.cs index e89c631f015f..b9810b23c2ee 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlockBlobStorageResource.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlockBlobStorageResource.cs @@ -30,16 +30,12 @@ internal class BlockBlobStorageResource : StorageResourceItemInternal /// private ConcurrentDictionary _blocks; - /// - /// The identifier for the type of storage resource. - /// protected override string ResourceId => "BlockBlob"; - /// - /// Gets the Uri of the StorageResource - /// public override Uri Uri => BlobClient.Uri; + public override string ProviderId => "blob"; + /// /// Defines the recommended Transfer Type of the storage resource. /// diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/PageBlobStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/PageBlobStorageResource.cs index 88737b707e98..6b7710fa25ec 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/PageBlobStorageResource.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/PageBlobStorageResource.cs @@ -20,16 +20,12 @@ internal class PageBlobStorageResource : StorageResourceItemInternal internal long? _length; internal ETag? _etagDownloadLock = default; - /// - /// The identifier for the type of storage resource. - /// protected override string ResourceId => "PageBlob"; - /// - /// Gets the Uri of the Storage Resource - /// public override Uri Uri => BlobClient.Uri; + public override string ProviderId => "blob"; + /// /// Defines the recommended Transfer Type for the storage resource. /// diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/RehydrateBlobResourceTests.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/RehydrateBlobResourceTests.cs index 0ff2fd389c6c..ea0c01252138 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/RehydrateBlobResourceTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/RehydrateBlobResourceTests.cs @@ -41,6 +41,18 @@ private static string ToResourceId(StorageResourceType type) }; } + private static string ToProviderId(StorageResourceType type) + { + return type switch + { + StorageResourceType.BlockBlob => "blob", + StorageResourceType.PageBlob => "blob", + StorageResourceType.AppendBlob => "blob", + StorageResourceType.Local => "local", + _ => throw new NotImplementedException(), + }; + } + private static Mock GetProperties( string checkpointerPath, string transferId, @@ -48,6 +60,8 @@ private static Mock GetProperties( string destinationPath, string sourceResourceId, string destinationResourceId, + string sourceProviderId, + string destinationProviderId, bool isContainer) { var mock = new Mock(MockBehavior.Strict); @@ -57,6 +71,8 @@ private static Mock GetProperties( mock.Setup(p => p.DestinationPath).Returns(destinationPath); mock.Setup(p => p.SourceTypeId).Returns(sourceResourceId); mock.Setup(p => p.DestinationTypeId).Returns(destinationResourceId); + mock.Setup(p => p.SourceProviderId).Returns(sourceProviderId); + mock.Setup(p => p.DestinationProviderId).Returns(destinationProviderId); mock.Setup(p => p.IsContainer).Returns(isContainer); return mock; } @@ -112,8 +128,10 @@ private async Task AddJobPartToCheckpointer( // Use mock resources that don't correspond to correct paths var sourceMock = new Mock(); sourceMock.Setup(s => s.Uri).Returns(new Uri(CheckpointerTesting.DefaultWebSourcePath)); + sourceMock.Setup(s => s.ProviderId).Returns(ToProviderId(sourceType)); var destMock = new Mock(); destMock.Setup(s => s.Uri).Returns(new Uri(CheckpointerTesting.DefaultWebDestinationPath)); + destMock.Setup(s => s.ProviderId).Returns(ToProviderId(destinationType)); await checkpointer.AddNewJobAsync(transferId, sourceMock.Object, destMock.Object); for (int currentPart = 0; currentPart < partCount; currentPart++) @@ -159,6 +177,8 @@ public async Task RehydrateBlockBlob( destinationPath, ToResourceId(sourceType), ToResourceId(destinationType), + ToProviderId(sourceType), + ToProviderId(destinationType), isContainer: false).Object; await AddJobPartToCheckpointer( @@ -196,6 +216,8 @@ public async Task RehydrateBlockBlob_Options() destinationPath, ToResourceId(sourceType), ToResourceId(destinationType), + ToProviderId(sourceType), + ToProviderId(destinationType), isContainer: false).Object; IDictionary metadata = DataProvider.BuildMetadata(); @@ -250,6 +272,8 @@ public async Task RehydratePageBlob( destinationPath, ToResourceId(sourceType), ToResourceId(destinationType), + ToProviderId(sourceType), + ToProviderId(destinationType), isContainer: false).Object; await AddJobPartToCheckpointer( @@ -287,6 +311,8 @@ public async Task RehydratePageBlob_Options() destinationPath, ToResourceId(sourceType), ToResourceId(destinationType), + ToProviderId(sourceType), + ToProviderId(destinationType), isContainer: false).Object; IDictionary metadata = DataProvider.BuildMetadata(); @@ -341,6 +367,8 @@ public async Task RehydrateAppendBlob( destinationPath, ToResourceId(sourceType), ToResourceId(destinationType), + ToProviderId(sourceType), + ToProviderId(destinationType), isContainer: false).Object; await AddJobPartToCheckpointer( @@ -378,6 +406,8 @@ public async Task RehydrateAppendBlob_Options() destinationPath, ToResourceId(sourceType), ToResourceId(destinationType), + ToProviderId(sourceType), + ToProviderId(destinationType), isContainer: false).Object; IDictionary metadata = DataProvider.BuildMetadata(); @@ -441,6 +471,8 @@ public async Task RehydrateBlobContainer( destinationParentPath, ToResourceId(sourceType), ToResourceId(destinationType), + ToProviderId(sourceType), + ToProviderId(destinationType), isContainer: true).Object; await AddJobPartToCheckpointer( diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/api/Azure.Storage.DataMovement.Files.Shares.net6.0.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/api/Azure.Storage.DataMovement.Files.Shares.net6.0.cs index 955bcdbd3f42..2c7c6cc1fad5 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/api/Azure.Storage.DataMovement.Files.Shares.net6.0.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/api/Azure.Storage.DataMovement.Files.Shares.net6.0.cs @@ -15,7 +15,7 @@ public ShareFilesStorageResourceProvider(Azure.Storage.DataMovement.Files.Shares public ShareFilesStorageResourceProvider(Azure.Storage.DataMovement.Files.Shares.ShareFilesStorageResourceProvider.GetStorageSharedKeyCredential getStorageSharedKeyCredentialAsync) { } public ShareFilesStorageResourceProvider(Azure.Storage.DataMovement.Files.Shares.ShareFilesStorageResourceProvider.GetTokenCredential getTokenCredentialAsync) { } public ShareFilesStorageResourceProvider(Azure.Storage.StorageSharedKeyCredential credential) { } - protected override string TypeId { get { throw null; } } + protected override string ProviderId { get { throw null; } } public Azure.Storage.DataMovement.StorageResource FromClient(Azure.Storage.Files.Shares.ShareDirectoryClient client, Azure.Storage.DataMovement.Files.Shares.ShareFileStorageResourceOptions options = null) { throw null; } public Azure.Storage.DataMovement.StorageResource FromClient(Azure.Storage.Files.Shares.ShareFileClient client, Azure.Storage.DataMovement.Files.Shares.ShareFileStorageResourceOptions options = null) { throw null; } protected override System.Threading.Tasks.Task FromDestinationAsync(Azure.Storage.DataMovement.DataTransferProperties properties, System.Threading.CancellationToken cancellationToken) { throw null; } diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/api/Azure.Storage.DataMovement.Files.Shares.netstandard2.0.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/api/Azure.Storage.DataMovement.Files.Shares.netstandard2.0.cs index 955bcdbd3f42..2c7c6cc1fad5 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/api/Azure.Storage.DataMovement.Files.Shares.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/api/Azure.Storage.DataMovement.Files.Shares.netstandard2.0.cs @@ -15,7 +15,7 @@ public ShareFilesStorageResourceProvider(Azure.Storage.DataMovement.Files.Shares public ShareFilesStorageResourceProvider(Azure.Storage.DataMovement.Files.Shares.ShareFilesStorageResourceProvider.GetStorageSharedKeyCredential getStorageSharedKeyCredentialAsync) { } public ShareFilesStorageResourceProvider(Azure.Storage.DataMovement.Files.Shares.ShareFilesStorageResourceProvider.GetTokenCredential getTokenCredentialAsync) { } public ShareFilesStorageResourceProvider(Azure.Storage.StorageSharedKeyCredential credential) { } - protected override string TypeId { get { throw null; } } + protected override string ProviderId { get { throw null; } } public Azure.Storage.DataMovement.StorageResource FromClient(Azure.Storage.Files.Shares.ShareDirectoryClient client, Azure.Storage.DataMovement.Files.Shares.ShareFileStorageResourceOptions options = null) { throw null; } public Azure.Storage.DataMovement.StorageResource FromClient(Azure.Storage.Files.Shares.ShareFileClient client, Azure.Storage.DataMovement.Files.Shares.ShareFileStorageResourceOptions options = null) { throw null; } protected override System.Threading.Tasks.Task FromDestinationAsync(Azure.Storage.DataMovement.DataTransferProperties properties, System.Threading.CancellationToken cancellationToken) { throw null; } diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryStorageResourceContainer.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryStorageResourceContainer.cs index 9a3ca0124f4e..161024ebb0fa 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryStorageResourceContainer.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryStorageResourceContainer.cs @@ -20,6 +20,8 @@ internal class ShareDirectoryStorageResourceContainer : StorageResourceContainer public override Uri Uri => ShareDirectoryClient.Uri; + public override string ProviderId => "share"; + internal ShareDirectoryStorageResourceContainer(ShareDirectoryClient shareDirectoryClient, ShareFileStorageResourceOptions options) { ShareDirectoryClient = shareDirectoryClient; diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs index 60262de884ce..819a094e74a4 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs @@ -22,6 +22,8 @@ internal class ShareFileStorageResource : StorageResourceItemInternal public override Uri Uri => ShareFileClient.Uri; + public override string ProviderId => "share"; + protected override string ResourceId => "ShareFile"; protected override DataTransferOrder TransferType => DataTransferOrder.Sequential; diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFilesStorageResourceProvider.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFilesStorageResourceProvider.cs index c832b48d3e27..9735d98f3d66 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFilesStorageResourceProvider.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFilesStorageResourceProvider.cs @@ -64,7 +64,7 @@ private enum CredentialType } /// - protected override string TypeId => "share"; + protected override string ProviderId => "share"; private readonly CredentialType _credentialType; private readonly GetStorageSharedKeyCredential _getStorageSharedKeyCredential; diff --git a/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.net6.0.cs b/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.net6.0.cs index 8e529a84e858..b021e5acfcdb 100644 --- a/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.net6.0.cs +++ b/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.net6.0.cs @@ -64,9 +64,11 @@ public partial class DataTransferProperties protected internal DataTransferProperties() { } public virtual Azure.Storage.DataMovement.TransferCheckpointStoreOptions Checkpointer { get { throw null; } } public virtual string DestinationPath { get { throw null; } } + public virtual string DestinationProviderId { get { throw null; } } public virtual string DestinationTypeId { get { throw null; } } public virtual bool IsContainer { get { throw null; } } public virtual string SourcePath { get { throw null; } } + public virtual string SourceProviderId { get { throw null; } } public virtual string SourceTypeId { get { throw null; } } public virtual string TransferId { get { throw null; } } } @@ -97,7 +99,7 @@ protected internal DataTransferStatus(Azure.Storage.DataMovement.DataTransferSta public partial class LocalFilesStorageResourceProvider : Azure.Storage.DataMovement.StorageResourceProvider { public LocalFilesStorageResourceProvider() { } - protected internal override string TypeId { get { throw null; } } + protected internal override string ProviderId { get { throw null; } } protected internal override System.Threading.Tasks.Task FromDestinationAsync(Azure.Storage.DataMovement.DataTransferProperties properties, System.Threading.CancellationToken cancellationToken) { throw null; } public Azure.Storage.DataMovement.StorageResourceContainer FromDirectory(string directoryPath) { throw null; } public Azure.Storage.DataMovement.StorageResourceItem FromFile(string filePath) { throw null; } @@ -113,6 +115,7 @@ public abstract partial class StorageResource { protected StorageResource() { } protected internal abstract bool IsContainer { get; } + public abstract string ProviderId { get; } public abstract System.Uri Uri { get; } } public abstract partial class StorageResourceCheckpointData @@ -168,7 +171,7 @@ public StorageResourceProperties(System.DateTimeOffset lastModified, System.Date public abstract partial class StorageResourceProvider { protected StorageResourceProvider() { } - protected internal abstract string TypeId { get; } + protected internal abstract string ProviderId { get; } protected internal abstract System.Threading.Tasks.Task FromDestinationAsync(Azure.Storage.DataMovement.DataTransferProperties properties, System.Threading.CancellationToken cancellationToken); protected internal abstract System.Threading.Tasks.Task FromSourceAsync(Azure.Storage.DataMovement.DataTransferProperties properties, System.Threading.CancellationToken cancellationToken); } diff --git a/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.netstandard2.0.cs b/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.netstandard2.0.cs index 8e529a84e858..b021e5acfcdb 100644 --- a/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.netstandard2.0.cs @@ -64,9 +64,11 @@ public partial class DataTransferProperties protected internal DataTransferProperties() { } public virtual Azure.Storage.DataMovement.TransferCheckpointStoreOptions Checkpointer { get { throw null; } } public virtual string DestinationPath { get { throw null; } } + public virtual string DestinationProviderId { get { throw null; } } public virtual string DestinationTypeId { get { throw null; } } public virtual bool IsContainer { get { throw null; } } public virtual string SourcePath { get { throw null; } } + public virtual string SourceProviderId { get { throw null; } } public virtual string SourceTypeId { get { throw null; } } public virtual string TransferId { get { throw null; } } } @@ -97,7 +99,7 @@ protected internal DataTransferStatus(Azure.Storage.DataMovement.DataTransferSta public partial class LocalFilesStorageResourceProvider : Azure.Storage.DataMovement.StorageResourceProvider { public LocalFilesStorageResourceProvider() { } - protected internal override string TypeId { get { throw null; } } + protected internal override string ProviderId { get { throw null; } } protected internal override System.Threading.Tasks.Task FromDestinationAsync(Azure.Storage.DataMovement.DataTransferProperties properties, System.Threading.CancellationToken cancellationToken) { throw null; } public Azure.Storage.DataMovement.StorageResourceContainer FromDirectory(string directoryPath) { throw null; } public Azure.Storage.DataMovement.StorageResourceItem FromFile(string filePath) { throw null; } @@ -113,6 +115,7 @@ public abstract partial class StorageResource { protected StorageResource() { } protected internal abstract bool IsContainer { get; } + public abstract string ProviderId { get; } public abstract System.Uri Uri { get; } } public abstract partial class StorageResourceCheckpointData @@ -168,7 +171,7 @@ public StorageResourceProperties(System.DateTimeOffset lastModified, System.Date public abstract partial class StorageResourceProvider { protected StorageResourceProvider() { } - protected internal abstract string TypeId { get; } + protected internal abstract string ProviderId { get; } protected internal abstract System.Threading.Tasks.Task FromDestinationAsync(Azure.Storage.DataMovement.DataTransferProperties properties, System.Threading.CancellationToken cancellationToken); protected internal abstract System.Threading.Tasks.Task FromSourceAsync(Azure.Storage.DataMovement.DataTransferProperties properties, System.Threading.CancellationToken cancellationToken); } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/CheckpointerExtensions.cs b/sdk/storage/Azure.Storage.DataMovement/src/CheckpointerExtensions.cs index 21aa786e8cd8..436b3efe1f20 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/CheckpointerExtensions.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/CheckpointerExtensions.cs @@ -44,14 +44,20 @@ internal static async Task GetDataTransferPropertiesAsyn string transferId, CancellationToken cancellationToken) { + JobPlanHeader header; + using (Stream stream = await checkpointer.ReadJobPlanFileAsync( + transferId, + offset: 0, + length: 0, // Read whole file + cancellationToken).ConfigureAwait(false)) + { + header = JobPlanHeader.Deserialize(stream); + } + (string sourceResourceId, string destResourceId) = await checkpointer.GetResourceIdsAsync( transferId, cancellationToken).ConfigureAwait(false); - (string sourcePath, string destPath) = await checkpointer.GetResourcePathsAsync( - transferId, - cancellationToken).ConfigureAwait(false); - bool isContainer = (await checkpointer.CurrentJobPartCountAsync(transferId, cancellationToken).ConfigureAwait(false)) > 1; @@ -59,9 +65,11 @@ internal static async Task GetDataTransferPropertiesAsyn { TransferId = transferId, SourceTypeId = sourceResourceId, - SourcePath = sourcePath, + SourcePath = header.ParentSourcePath, + SourceProviderId = header.SourceProviderId, DestinationTypeId = destResourceId, - DestinationPath = destPath, + DestinationPath = header.ParentDestinationPath, + DestinationProviderId = header.DestinationProviderId, IsContainer = isContainer, }; } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/DataTransferProperties.cs b/sdk/storage/Azure.Storage.DataMovement/src/DataTransferProperties.cs index 369887dd736a..fafbe32a1788 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/DataTransferProperties.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/DataTransferProperties.cs @@ -28,6 +28,11 @@ public class DataTransferProperties /// public virtual string SourcePath { get; internal set; } + /// + /// A string ID for the source resource provider that should be used for rehydration. + /// + public virtual string SourceProviderId { get; internal set; } + /// /// Contains the Source Scheme of the Storage Resource to rehydrate the StorageResource from. /// @@ -38,6 +43,11 @@ public class DataTransferProperties /// public virtual string DestinationPath { get; internal set; } + /// + /// A string ID for the destination resource provider that should be used for rehydration. + /// + public virtual string DestinationProviderId { get; internal set; } + /// /// Defines whether or not this was a container transfer, in order to rehydrate the StorageResource. /// diff --git a/sdk/storage/Azure.Storage.DataMovement/src/LocalDirectoryStorageResourceContainer.cs b/sdk/storage/Azure.Storage.DataMovement/src/LocalDirectoryStorageResourceContainer.cs index 8180061e17a1..4afa49c03dae 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/LocalDirectoryStorageResourceContainer.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/LocalDirectoryStorageResourceContainer.cs @@ -6,7 +6,6 @@ using System.IO; using System.Runtime.CompilerServices; using System.Threading; -using System.Threading.Tasks; using Azure.Core; namespace Azure.Storage.DataMovement @@ -18,11 +17,10 @@ internal class LocalDirectoryStorageResourceContainer : StorageResourceContainer { private Uri _uri; - /// - /// Gets the path - /// public override Uri Uri => _uri; + public override string ProviderId => "local"; + /// /// Constructor /// diff --git a/sdk/storage/Azure.Storage.DataMovement/src/LocalFileStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement/src/LocalFileStorageResource.cs index 9501b4eaa07f..4afabe93e51c 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/LocalFileStorageResource.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/LocalFileStorageResource.cs @@ -16,16 +16,12 @@ internal class LocalFileStorageResource : StorageResourceItem { private Uri _uri; - /// - /// The identifier for the type of storage resource. - /// protected internal override string ResourceId => "LocalFile"; - /// - /// Gets the Uri of the resource. - /// public override Uri Uri => _uri; + public override string ProviderId => "local"; + /// /// Defines the recommended Transfer Type of the resource /// diff --git a/sdk/storage/Azure.Storage.DataMovement/src/LocalFilesStorageResourceProvider.cs b/sdk/storage/Azure.Storage.DataMovement/src/LocalFilesStorageResourceProvider.cs index 610004e09461..0630517ea7ef 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/LocalFilesStorageResourceProvider.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/LocalFilesStorageResourceProvider.cs @@ -14,7 +14,7 @@ namespace Azure.Storage.DataMovement public class LocalFilesStorageResourceProvider : StorageResourceProvider { /// - protected internal override string TypeId => "LocalFile"; + protected internal override string ProviderId => "local"; /// /// Default constructor. diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataMovementConstants.cs b/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataMovementConstants.cs index 45a72ffaefa0..d1f08f199e3c 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataMovementConstants.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataMovementConstants.cs @@ -72,12 +72,16 @@ internal static class JobPlanFile internal const int VersionStrLength = 2; internal const int VersionStrNumBytes = VersionStrLength * 2; + internal const int ProviderIdMaxLength = 5; + internal const int ProviderIdNumBytes = ProviderIdMaxLength * 2; internal const int VersionIndex = 0; internal const int TransferIdIndex = VersionIndex + VersionStrNumBytes; internal const int CrateTimeIndex = TransferIdIndex + GuidSizeInBytes; internal const int OperationTypeIndex = CrateTimeIndex + LongSizeInBytes; - internal const int EnumerationCompleteIndex = OperationTypeIndex + OneByte; + internal const int SourceProviderIdIndex = OperationTypeIndex + OneByte; + internal const int DestinationProviderIdIndex = SourceProviderIdIndex + ProviderIdNumBytes; + internal const int EnumerationCompleteIndex = DestinationProviderIdIndex + ProviderIdNumBytes; internal const int JobStatusIndex = EnumerationCompleteIndex + OneByte; internal const int ParentSourcePathOffsetIndex = JobStatusIndex + IntSizeInBytes; internal const int ParentSourcePathLengthIndex = ParentSourcePathOffsetIndex + IntSizeInBytes; diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/Errors.DataMovement.cs b/sdk/storage/Azure.Storage.DataMovement/src/Shared/Errors.DataMovement.cs index d29515d58494..69a9fd5bf724 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/Shared/Errors.DataMovement.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/Shared/Errors.DataMovement.cs @@ -124,5 +124,9 @@ public static InvalidOperationException InvalidTransferResourceTypes() public static ArgumentException ResourceUriInvalid(string parameterResource) => new ArgumentException($"Could not perform operation because {parameterResource} was expected to be not a Local Storage Resource."); + + public static ArgumentException NoResourceProviderFound(bool isSource, string providerId) + => new ArgumentException($"Unable to find resource provider for transfer {(isSource ? "source" : "destination")} with provider id: {providerId}. " + + $"Please ensure you have registered the required resource provider with TransferManagerOptions.ResumeProviders."); } } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPlanHeader.cs b/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPlanHeader.cs index b8e4ffebecd2..e87e6f545652 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPlanHeader.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPlanHeader.cs @@ -30,6 +30,16 @@ internal class JobPlanHeader /// public JobPlanOperation OperationType; + /// + /// A string ID of the source resource provider to use for rehydration. + /// + public string SourceProviderId; + + /// + /// A string ID of the destination resource provider to use for rehydration. + /// + public string DestinationProviderId; + /// /// Whether or not the enumeration of the parent container has completed. /// @@ -55,6 +65,8 @@ public JobPlanHeader( string transferId, DateTimeOffset createTime, JobPlanOperation operationType, + string sourceProviderId, + string destinationProviderId, bool enumerationComplete, DataTransferStatus jobStatus, string parentSourcePath, @@ -62,14 +74,27 @@ public JobPlanHeader( { Argument.AssertNotNull(version, nameof(version)); Argument.AssertNotNullOrEmpty(transferId, nameof(transferId)); + Argument.AssertNotNullOrEmpty(sourceProviderId, nameof(sourceProviderId)); + Argument.AssertNotNullOrEmpty(destinationProviderId, nameof(destinationProviderId)); Argument.AssertNotNull(createTime, nameof(createTime)); Argument.AssertNotNullOrEmpty(parentSourcePath, nameof(parentSourcePath)); Argument.AssertNotNullOrEmpty(parentDestinationPath, nameof(parentDestinationPath)); + if (sourceProviderId.Length > DataMovementConstants.JobPlanFile.ProviderIdMaxLength) + { + throw new ArgumentException("The provided sourceProviderId is too long."); + } + if (destinationProviderId.Length > DataMovementConstants.JobPlanFile.ProviderIdMaxLength) + { + throw new ArgumentException("The provided destinationProviderId is too long."); + } + Version = version; TransferId = transferId; CreateTime = createTime; OperationType = operationType; + SourceProviderId = sourceProviderId; + DestinationProviderId = destinationProviderId; EnumerationComplete = enumerationComplete; JobStatus = jobStatus; ParentSourcePath = parentSourcePath; @@ -96,6 +121,12 @@ public void Serialize(Stream stream) // OperationType writer.Write((byte)OperationType); + // SourceProviderId + WritePaddedString(writer, SourceProviderId, DataMovementConstants.JobPlanFile.ProviderIdNumBytes); + + // DestinationProviderId + WritePaddedString(writer, DestinationProviderId, DataMovementConstants.JobPlanFile.ProviderIdNumBytes); + // EnumerationComplete writer.Write(Convert.ToByte(EnumerationComplete)); @@ -143,6 +174,12 @@ public static JobPlanHeader Deserialize(Stream stream) byte operationTypeByte = reader.ReadByte(); JobPlanOperation operationType = (JobPlanOperation)operationTypeByte; + // SourceProviderId + string sourceProviderId = ReadPaddedString(reader, DataMovementConstants.JobPlanFile.ProviderIdNumBytes); + + // DestinationProviderId + string destProviderId = ReadPaddedString(reader, DataMovementConstants.JobPlanFile.ProviderIdNumBytes); + // EnumerationComplete byte enumerationCompleteByte = reader.ReadByte(); bool enumerationComplete = Convert.ToBoolean(enumerationCompleteByte); @@ -185,6 +222,8 @@ public static JobPlanHeader Deserialize(Stream stream) transferId, createTime, operationType, + sourceProviderId, + destProviderId, enumerationComplete, jobPlanStatus.ToDataTransferStatus(), parentSourcePath, @@ -204,6 +243,12 @@ private static void WritePaddedString(BinaryWriter writer, string value, int set } } + private static string ReadPaddedString(BinaryReader reader, int numBytes) + { + byte[] stringBytes = reader.ReadBytes(numBytes); + return stringBytes.ToString(numBytes).TrimEnd('\0'); + } + private static void CheckSchemaVersion(string version) { if (version != DataMovementConstants.JobPlanFile.SchemaVersion) diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlanExtensions.cs b/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlanExtensions.cs index 59101bfbf938..12e9c86b4e8e 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlanExtensions.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlanExtensions.cs @@ -46,51 +46,6 @@ internal static JobPartPlanHeader GetJobPartPlanHeader(this JobPartPlanFileName return result; } - internal static async Task<(string Source, string Destination)> GetResourcePathsAsync( - this TransferCheckpointer checkpointer, - string transferId, - CancellationToken cancellationToken) - { - int startIndex = DataMovementConstants.JobPlanFile.ParentSourcePathOffsetIndex; - - string parentSourcePath = default; - string parentDestinationPath = default; - using (Stream stream = await checkpointer.ReadJobPlanFileAsync( - transferId: transferId, - offset: startIndex, - length: 0, // Read to the end - cancellationToken: cancellationToken).ConfigureAwait(false)) - { - BinaryReader reader = new BinaryReader(stream); - - // ParentSourcePath offset/length - int parentSourcePathOffset = reader.ReadInt32() - startIndex; - int parentSourcePathLength = reader.ReadInt32(); - - // ParentDestinationPath offset/length - int parentDestinationPathOffset = reader.ReadInt32() - startIndex; - int parentDestinationPathLength = reader.ReadInt32(); - - // ParentSourcePath - if (parentSourcePathOffset > 0) - { - reader.BaseStream.Position = parentSourcePathOffset; - byte[] parentSourcePathBytes = reader.ReadBytes(parentSourcePathLength); - parentSourcePath = parentSourcePathBytes.ToString(parentSourcePathLength); - } - - // ParentDestinationPath - if (parentDestinationPathOffset > 0) - { - reader.BaseStream.Position = parentDestinationPathOffset; - byte[] parentDestinationPathBytes = reader.ReadBytes(parentDestinationPathLength); - parentDestinationPath = parentDestinationPathBytes.ToString(parentDestinationPathLength); - } - } - - return (parentSourcePath, parentDestinationPath); - } - internal static async Task<(string Source, string Destination)> GetResourceIdsAsync( this TransferCheckpointer checkpointer, string transferId, diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/LocalTransferCheckpointer.cs b/sdk/storage/Azure.Storage.DataMovement/src/Shared/LocalTransferCheckpointer.cs index 8de5ea3ab7bf..1cb7dfc86488 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/Shared/LocalTransferCheckpointer.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/Shared/LocalTransferCheckpointer.cs @@ -72,6 +72,8 @@ public override async Task AddNewJobAsync( transferId, DateTimeOffset.UtcNow, GetOperationType(source, destination), + source.ProviderId, + destination.ProviderId, false, /* enumerationComplete */ new DataTransferStatusInternal(), source.Uri.AbsoluteUri, diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/TransferManager.cs b/sdk/storage/Azure.Storage.DataMovement/src/Shared/TransferManager.cs index b5e6dc45a952..94d625a21ea6 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/Shared/TransferManager.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/Shared/TransferManager.cs @@ -371,7 +371,7 @@ bool TryGetStorageResourceProvider(DataTransferProperties properties, bool getSo { foreach (StorageResourceProvider provider in _resumeProviders) { - if (provider.TypeId == (getSource ? properties.SourceTypeId : properties.DestinationTypeId)) + if (provider.ProviderId == (getSource ? properties.SourceProviderId : properties.DestinationProviderId)) { resourceProvider = provider; return true; @@ -392,11 +392,11 @@ bool TryGetStorageResourceProvider(DataTransferProperties properties, bool getSo if (!TryGetStorageResourceProvider(dataTransferProperties, getSource: true, out StorageResourceProvider sourceProvider)) { - throw new Exception(); + throw Errors.NoResourceProviderFound(true, dataTransferProperties.SourceProviderId); } if (!TryGetStorageResourceProvider(dataTransferProperties, getSource: false, out StorageResourceProvider destinationProvider)) { - throw new Exception(); + throw Errors.NoResourceProviderFound(false, dataTransferProperties.DestinationProviderId); } DataTransfer dataTransfer = await BuildAndAddTransferJobAsync( diff --git a/sdk/storage/Azure.Storage.DataMovement/src/StorageResource.cs b/sdk/storage/Azure.Storage.DataMovement/src/StorageResource.cs index 566c5f8dd85b..cc04d0c37eec 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/StorageResource.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/StorageResource.cs @@ -26,5 +26,11 @@ protected StorageResource() /// Gets the Uri of the Storage Resource. /// public abstract Uri Uri { get; } + + /// + /// A string ID for the resource provider that should be used for rehydration. + /// NOTE: Must be no more than 5 characters long. + /// + public abstract string ProviderId { get; } } } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceItem.cs b/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceItem.cs index 7442e51567e1..fb9f0d2bbbf0 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceItem.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceItem.cs @@ -4,7 +4,6 @@ using System.IO; using System.Threading; using System.Threading.Tasks; -using Azure.Storage.DataMovement.JobPlan; namespace Azure.Storage.DataMovement { diff --git a/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceProvider.cs b/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceProvider.cs index ac2280e59b77..76e3120de714 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceProvider.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceProvider.cs @@ -12,9 +12,9 @@ namespace Azure.Storage.DataMovement public abstract class StorageResourceProvider { /// - /// Type ID. For DataMovement to query in selecting appropirate provider on resume. + /// Provider ID. For DataMovement to query in selecting appropriate provider on resume. /// - protected internal abstract string TypeId { get; } + protected internal abstract string ProviderId { get; } /// /// Gets a source resource from the given transfer properties. diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/GetTransfersTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/GetTransfersTests.cs index 27b7430b2b9d..fbbcf80c57a2 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/GetTransfersTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/GetTransfersTests.cs @@ -243,11 +243,11 @@ public async Task GetResumableTransfers_LocalCheckpointer() // Build expected results first to use to populate checkpointer DataTransferProperties[] expectedResults = new DataTransferProperties[] { - new DataTransferProperties { TransferId = Guid.NewGuid().ToString(), SourceTypeId = "LocalFile", SourcePath = parentLocalPath1 + "file1", DestinationTypeId = "BlockBlob", DestinationPath = parentRemotePath + "file1", IsContainer = false }, - new DataTransferProperties { TransferId = Guid.NewGuid().ToString(), SourceTypeId = "BlockBlob", SourcePath = parentRemotePath + "file2/", DestinationTypeId = "LocalFile", DestinationPath = parentLocalPath1 + "file2/", IsContainer = false }, - new DataTransferProperties { TransferId = Guid.NewGuid().ToString(), SourceTypeId = "BlockBlob", SourcePath = parentRemotePath + "file3", DestinationTypeId = "BlockBlob", DestinationPath = parentRemotePath + "file3", IsContainer = false }, - new DataTransferProperties { TransferId = Guid.NewGuid().ToString(), SourceTypeId = "BlockBlob", SourcePath = parentRemotePath, DestinationTypeId = "LocalFile", DestinationPath = parentLocalPath1, IsContainer = true }, - new DataTransferProperties { TransferId = Guid.NewGuid().ToString(), SourceTypeId = "LocalFile", SourcePath = parentLocalPath2, DestinationTypeId = "AppendBlob", DestinationPath = parentRemotePath, IsContainer = true }, + new DataTransferProperties { TransferId = Guid.NewGuid().ToString(), SourceProviderId = "local", SourceTypeId = "LocalFile", SourcePath = parentLocalPath1 + "file1", DestinationProviderId = "blob", DestinationTypeId = "BlockBlob", DestinationPath = parentRemotePath + "file1", IsContainer = false }, + new DataTransferProperties { TransferId = Guid.NewGuid().ToString(), SourceProviderId = "blob", SourceTypeId = "BlockBlob", SourcePath = parentRemotePath + "file2/", DestinationProviderId = "local", DestinationTypeId = "LocalFile", DestinationPath = parentLocalPath1 + "file2/", IsContainer = false }, + new DataTransferProperties { TransferId = Guid.NewGuid().ToString(), SourceProviderId = "blob", SourceTypeId = "BlockBlob", SourcePath = parentRemotePath + "file3", DestinationProviderId = "blob", DestinationTypeId = "BlockBlob", DestinationPath = parentRemotePath + "file3", IsContainer = false }, + new DataTransferProperties { TransferId = Guid.NewGuid().ToString(), SourceProviderId = "blob", SourceTypeId = "BlockBlob", SourcePath = parentRemotePath, DestinationProviderId = "local", DestinationTypeId = "LocalFile", DestinationPath = parentLocalPath1, IsContainer = true }, + new DataTransferProperties { TransferId = Guid.NewGuid().ToString(), SourceProviderId = "local", SourceTypeId = "LocalFile", SourcePath = parentLocalPath2, DestinationProviderId = "blob", DestinationTypeId = "AppendBlob", DestinationPath = parentRemotePath, IsContainer = true }, }; // Add a transfer for each expected result @@ -320,7 +320,9 @@ private void AddTransferFromDataTransferProperties( checkpointerPath, properties.TransferId, parentSourcePath: properties.SourcePath, - parentDestinationPath: properties.DestinationPath); + parentDestinationPath: properties.DestinationPath, + sourceProviderId: properties.SourceProviderId, + destinationProviderId: properties.DestinationProviderId); if (properties.IsContainer) { @@ -368,8 +370,10 @@ private void AddTransferFromDataTransferProperties( private void AssertTransferProperties(DataTransferProperties expected, DataTransferProperties actual) { Assert.AreEqual(expected.TransferId, actual.TransferId); + Assert.AreEqual(expected.SourceProviderId, actual.SourceProviderId); Assert.AreEqual(expected.SourceTypeId, actual.SourceTypeId); Assert.AreEqual(expected.SourcePath.TrimEnd('\\', '/'), actual.SourcePath.TrimEnd('\\', '/')); + Assert.AreEqual(expected.DestinationProviderId, actual.DestinationProviderId); Assert.AreEqual(expected.DestinationTypeId, actual.DestinationTypeId); Assert.AreEqual(expected.DestinationPath.TrimEnd('\\', '/'), actual.DestinationPath.TrimEnd('\\', '/')); Assert.AreEqual(expected.IsContainer, actual.IsContainer); diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/JobPlanHeaderTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/JobPlanHeaderTests.cs index 147476562e1f..156a3879b2ad 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/JobPlanHeaderTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/JobPlanHeaderTests.cs @@ -23,6 +23,8 @@ public void Ctor() Assert.AreEqual(DefaultTransferId, header.TransferId); Assert.AreEqual(DefaultCreateTime, header.CreateTime); Assert.AreEqual(DefaultJobPlanOperation, header.OperationType); + Assert.AreEqual(DefaultSourceProviderId, header.SourceProviderId); + Assert.AreEqual(DefaultDestinationProviderId, header.DestinationProviderId); Assert.AreEqual(false, header.EnumerationComplete); Assert.AreEqual(DefaultJobStatus, header.JobStatus); Assert.AreEqual(DefaultSourcePath, header.ParentSourcePath); @@ -41,7 +43,7 @@ public void Serialize() header.Serialize(headerStream); BinaryReader reader = new(fileStream); - byte[] expected = reader.ReadBytes((int) fileStream.Length); + byte[] expected = reader.ReadBytes((int)fileStream.Length); byte[] actual = headerStream.ToArray(); CollectionAssert.AreEqual(expected, actual); @@ -77,6 +79,8 @@ private void DeserializeAndVerify(Stream stream, string version) Assert.AreEqual(DefaultTransferId, deserialized.TransferId); Assert.AreEqual(DefaultCreateTime, deserialized.CreateTime); Assert.AreEqual(DefaultJobPlanOperation, deserialized.OperationType); + Assert.AreEqual(DefaultSourceProviderId, deserialized.SourceProviderId); + Assert.AreEqual(DefaultDestinationProviderId, deserialized.DestinationProviderId); Assert.AreEqual(false, deserialized.EnumerationComplete); Assert.AreEqual(DefaultJobStatus, deserialized.JobStatus); Assert.AreEqual(DefaultSourcePath, deserialized.ParentSourcePath); diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/LocalTransferCheckpointerFactory.cs b/sdk/storage/Azure.Storage.DataMovement/tests/LocalTransferCheckpointerFactory.cs index e5956e9804c5..73e7533bf5a5 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/LocalTransferCheckpointerFactory.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/LocalTransferCheckpointerFactory.cs @@ -17,9 +17,11 @@ internal class LocalTransferCheckpointerFactory internal const long _testPartNumber = 5; internal static readonly DateTimeOffset _testStartTime = new DateTimeOffset(2023, 03, 13, 15, 24, 6, default); + internal const string _testSourceProviderId = "test"; internal const string _testSourceResourceId = "LocalFile"; internal const string _testSourcePath = "C:/sample-source"; internal const string _testSourceQuery = "sourcequery"; + internal const string _testDestinationProviderId = "test"; internal const string _testDestinationResourceId = "LocalFile"; internal const string _testDestinationPath = "C:/sample-destination"; internal const string _testDestinationQuery = "destquery"; @@ -148,6 +150,8 @@ internal void CreateStubJobPlanFile( string transferId, string parentSourcePath = _testSourcePath, string parentDestinationPath = _testDestinationPath, + string sourceProviderId = _testSourceProviderId, + string destinationProviderId = _testDestinationProviderId, DataTransferStatus status = default) { status ??= new DataTransferStatus(); @@ -156,6 +160,8 @@ internal void CreateStubJobPlanFile( transferId, DateTimeOffset.UtcNow, JobPlanOperation.ServiceToService, + sourceProviderId, + destinationProviderId, false, /* enumerationComplete */ status, parentSourcePath, diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/MockStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement/tests/MockStorageResource.cs index fa38f02fd31f..fafdd03f10fa 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/MockStorageResource.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/MockStorageResource.cs @@ -16,6 +16,8 @@ internal class MockStorageResource : StorageResourceItem public override Uri Uri => _uri; + public override string ProviderId => "mock"; + protected internal override string ResourceId => "Mock"; protected internal override DataTransferOrder TransferType => DataTransferOrder.Sequential; diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Resources/SampleJobPlanFile.b1.ndm b/sdk/storage/Azure.Storage.DataMovement/tests/Resources/SampleJobPlanFile.b1.ndm index 2ce5cb307dbf..07cf481e1571 100644 Binary files a/sdk/storage/Azure.Storage.DataMovement/tests/Resources/SampleJobPlanFile.b1.ndm and b/sdk/storage/Azure.Storage.DataMovement/tests/Resources/SampleJobPlanFile.b1.ndm differ diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/CheckpointerTesting.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/CheckpointerTesting.cs index f9b335ea9fe4..2235d0d96a4c 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/CheckpointerTesting.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/CheckpointerTesting.cs @@ -26,10 +26,12 @@ internal class CheckpointerTesting internal const long DefaultPartNumber = 5; internal static readonly DateTimeOffset DefaultStartTime = new DateTimeOffset(2023, 03, 13, 15, 24, 6, default); + internal const string DefaultSourceProviderId = "test"; internal const string DefaultSourceResourceId = "LocalFile"; internal const string DefaultSourcePath = "C:/sample-source"; internal const string DefaultWebSourcePath = "https://example.com/source"; internal const string DefaultSourceQuery = "sourcequery"; + internal const string DefaultDestinationProviderId = "test"; internal const string DefaultDestinationResourceId = "LocalFile"; internal const string DefaultDestinationPath = "C:/sample-destination"; internal const string DefaultWebDestinationPath = "https://example.com/destination"; @@ -181,6 +183,8 @@ internal static JobPlanHeader CreateDefaultJobHeader( string transferId = DefaultTransferId, DateTimeOffset createTime = default, JobPlanOperation operationType = DefaultJobPlanOperation, + string sourceProviderId = DefaultSourceProviderId, + string destinationProviderId = DefaultDestinationProviderId, bool enumerationComplete = false, DataTransferStatus jobStatus = default, string parentSourcePath = DefaultSourcePath, @@ -197,6 +201,8 @@ internal static JobPlanHeader CreateDefaultJobHeader( transferId, createTime, operationType, + sourceProviderId, + destinationProviderId, enumerationComplete, jobStatus, parentSourcePath, diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryStorageResourceContainer.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryStorageResourceContainer.cs index 227f3faf3781..f40571647fc9 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryStorageResourceContainer.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryStorageResourceContainer.cs @@ -18,6 +18,8 @@ internal class MemoryStorageResourceContainer : StorageResourceContainer public override Uri Uri { get; } + public override string ProviderId => "mock"; + public MemoryStorageResourceContainer(Uri uri) { Uri = uri ?? new Uri($"memory://localhost/mycontainer/mypath-{Guid.NewGuid()}/resource-item-{Guid.NewGuid()}"); diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryStorageResourceItem.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryStorageResourceItem.cs index eae69e6cc9b0..7363eaba93d1 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryStorageResourceItem.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryStorageResourceItem.cs @@ -14,6 +14,8 @@ internal class MemoryStorageResourceItem : StorageResourceItem public override Uri Uri { get; } + public override string ProviderId => "mock"; + protected internal override string ResourceId => "MemoryBuffer"; protected internal override DataTransferOrder TransferType => DataTransferOrder.Unordered;