diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/AppendBlobStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/AppendBlobStorageResource.cs index efa9243fe3f8..12bdf35dcf14 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/AppendBlobStorageResource.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/AppendBlobStorageResource.cs @@ -283,12 +283,12 @@ protected override async Task DeleteIfExistsAsync(CancellationToken cancel return await BlobClient.DeleteIfExistsAsync(cancellationToken: cancellationToken).ConfigureAwait(false); } - public override StorageResourceCheckpointData GetSourceCheckpointData() + protected override StorageResourceCheckpointData GetSourceCheckpointData() { return new BlobSourceCheckpointData(BlobType.Append); } - public override StorageResourceCheckpointData GetDestinationCheckpointData() + protected override StorageResourceCheckpointData GetDestinationCheckpointData() { return new BlobDestinationCheckpointData( BlobType.Append, diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/Azure.Storage.DataMovement.Blobs.csproj b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/Azure.Storage.DataMovement.Blobs.csproj index f76f95e0baeb..fd35a3bc2933 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/Azure.Storage.DataMovement.Blobs.csproj +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/Azure.Storage.DataMovement.Blobs.csproj @@ -47,15 +47,9 @@ - - - - - - diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobDestinationCheckpointData.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobDestinationCheckpointData.cs index 75ae4edbf031..f229d31e47dd 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobDestinationCheckpointData.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobDestinationCheckpointData.cs @@ -6,7 +6,6 @@ using System.Text; using Azure.Core; using Azure.Storage.Blobs.Models; -using static Azure.Storage.DataMovement.JobPlanExtensions; using Metadata = System.Collections.Generic.IDictionary; using Tags = System.Collections.Generic.IDictionary; @@ -73,7 +72,7 @@ public BlobDestinationCheckpointData( _cpkScopeBytes = CpkScope != default ? Encoding.UTF8.GetBytes(CpkScope) : Array.Empty(); } - public override void Serialize(Stream stream) + protected override void Serialize(Stream stream) { Argument.AssertNotNull(stream, nameof(stream)); @@ -87,31 +86,31 @@ public override void Serialize(Stream stream) writer.Write((byte)BlobType); // ContentType offset/length - WriteVariableLengthFieldInfo(writer, _contentTypeBytes.Length, ref currentVariableLengthIndex); + writer.WriteVariableLengthFieldInfo(_contentTypeBytes.Length, ref currentVariableLengthIndex); // ContentEncoding offset/length - WriteVariableLengthFieldInfo(writer, _contentEncodingBytes.Length, ref currentVariableLengthIndex); + writer.WriteVariableLengthFieldInfo(_contentEncodingBytes.Length, ref currentVariableLengthIndex); // ContentLanguage offset/length - WriteVariableLengthFieldInfo(writer, _contentLanguageBytes.Length, ref currentVariableLengthIndex); + writer.WriteVariableLengthFieldInfo(_contentLanguageBytes.Length, ref currentVariableLengthIndex); // ContentDisposition offset/length - WriteVariableLengthFieldInfo(writer, _contentDispositionBytes.Length, ref currentVariableLengthIndex); + writer.WriteVariableLengthFieldInfo(_contentDispositionBytes.Length, ref currentVariableLengthIndex); // CacheControl offset/length - WriteVariableLengthFieldInfo(writer, _cacheControlBytes.Length, ref currentVariableLengthIndex); + writer.WriteVariableLengthFieldInfo(_cacheControlBytes.Length, ref currentVariableLengthIndex); // AccessTier writer.Write((byte)AccessTier.ToJobPlanAccessTier()); // Metadata offset/length - WriteVariableLengthFieldInfo(writer, _metadataBytes.Length, ref currentVariableLengthIndex); + writer.WriteVariableLengthFieldInfo(_metadataBytes.Length, ref currentVariableLengthIndex); // Tags offset/length - WriteVariableLengthFieldInfo(writer, _tagsBytes.Length, ref currentVariableLengthIndex); + writer.WriteVariableLengthFieldInfo(_tagsBytes.Length, ref currentVariableLengthIndex); // CpkScope offset/length - WriteVariableLengthFieldInfo(writer, _cpkScopeBytes.Length, ref currentVariableLengthIndex); + writer.WriteVariableLengthFieldInfo(_cpkScopeBytes.Length, ref currentVariableLengthIndex); writer.Write(_contentTypeBytes); writer.Write(_contentEncodingBytes); diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobSourceCheckpointData.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobSourceCheckpointData.cs index 83170b9ed972..3ca1375150ec 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobSourceCheckpointData.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobSourceCheckpointData.cs @@ -16,7 +16,7 @@ public BlobSourceCheckpointData(BlobType blobType) public override int Length => DataMovementBlobConstants.SourceCheckpointData.DataSize; - public override void Serialize(Stream stream) + protected override void Serialize(Stream stream) { Argument.AssertNotNull(stream, nameof(stream)); BinaryWriter writer = new BinaryWriter(stream); diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs index 02ea5e1f1fef..89bdc27d0216 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs @@ -129,13 +129,13 @@ protected override async IAsyncEnumerable GetStorageResourcesAs } } - public override StorageResourceCheckpointData GetSourceCheckpointData() + protected override StorageResourceCheckpointData GetSourceCheckpointData() { // Source blob type does not matter for container return new BlobSourceCheckpointData(BlobType.Block); } - public override StorageResourceCheckpointData GetDestinationCheckpointData() + protected override StorageResourceCheckpointData GetDestinationCheckpointData() { return new BlobDestinationCheckpointData( _options?.BlobType ?? BlobType.Block, diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlockBlobStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlockBlobStorageResource.cs index 86fa48ce6c78..24b67522d0b4 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlockBlobStorageResource.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlockBlobStorageResource.cs @@ -325,12 +325,12 @@ protected override async Task DeleteIfExistsAsync(CancellationToken cancel return await BlobClient.DeleteIfExistsAsync(cancellationToken: cancellationToken).ConfigureAwait(false); } - public override StorageResourceCheckpointData GetSourceCheckpointData() + protected override StorageResourceCheckpointData GetSourceCheckpointData() { return new BlobSourceCheckpointData(BlobType.Block); } - public override StorageResourceCheckpointData GetDestinationCheckpointData() + protected override StorageResourceCheckpointData GetDestinationCheckpointData() { return new BlobDestinationCheckpointData( BlobType.Block, diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/PageBlobStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/PageBlobStorageResource.cs index 660e0d921951..0f90024b6c0e 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/PageBlobStorageResource.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/PageBlobStorageResource.cs @@ -287,12 +287,12 @@ protected override async Task DeleteIfExistsAsync(CancellationToken cancel return await BlobClient.DeleteIfExistsAsync(cancellationToken: cancellationToken).ConfigureAwait(false); } - public override StorageResourceCheckpointData GetSourceCheckpointData() + protected override StorageResourceCheckpointData GetSourceCheckpointData() { return new BlobSourceCheckpointData(BlobType.Page); } - public override StorageResourceCheckpointData GetDestinationCheckpointData() + protected override StorageResourceCheckpointData GetDestinationCheckpointData() { return new BlobDestinationCheckpointData( BlobType.Page, diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/Azure.Storage.DataMovement.Blobs.Tests.csproj b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/Azure.Storage.DataMovement.Blobs.Tests.csproj index 68c679cfb16e..af1899657987 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/Azure.Storage.DataMovement.Blobs.Tests.csproj +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/Azure.Storage.DataMovement.Blobs.Tests.csproj @@ -16,7 +16,6 @@ - diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/RehydrateBlobResourceTests.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/RehydrateBlobResourceTests.cs index 5b61e6bf7df2..da4b35c6175f 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/RehydrateBlobResourceTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/RehydrateBlobResourceTests.cs @@ -28,17 +28,6 @@ private enum StorageResourceType Local } - private static string ToResourceId(StorageResourceType type) - { - return type switch - { - StorageResourceType.BlockBlob => "BlockBlob", - StorageResourceType.PageBlob => "PageBlob", - StorageResourceType.AppendBlob => "AppendBlob", - _ => throw new NotImplementedException(), - }; - } - private static string ToProviderId(StorageResourceType type) { return type switch @@ -91,12 +80,9 @@ private static byte[] GetBytes(BlobCheckpointData checkpointData) } private static Mock GetProperties( - string checkpointerPath, string transferId, string sourcePath, string destinationPath, - string sourceResourceId, - string destinationResourceId, string sourceProviderId, string destinationProviderId, bool isContainer, @@ -105,11 +91,8 @@ private static Mock GetProperties( { var mock = new Mock(MockBehavior.Strict); mock.Setup(p => p.TransferId).Returns(transferId); - mock.Setup(p => p.Checkpointer).Returns(new TransferCheckpointStoreOptions(checkpointerPath)); mock.Setup(p => p.SourceUri).Returns(new Uri(sourcePath)); mock.Setup(p => p.DestinationUri).Returns(new Uri(destinationPath)); - mock.Setup(p => p.SourceTypeId).Returns(sourceResourceId); - mock.Setup(p => p.DestinationTypeId).Returns(destinationResourceId); mock.Setup(p => p.SourceProviderId).Returns(sourceProviderId); mock.Setup(p => p.DestinationProviderId).Returns(destinationProviderId); mock.Setup(p => p.SourceCheckpointData).Returns(GetBytes(sourceCheckpointData)); @@ -122,7 +105,6 @@ private static Mock GetProperties( public async Task RehydrateBlockBlob( [Values(true, false)] bool isSource) { - using DisposingLocalDirectory test = DisposingLocalDirectory.GetTestDirectory(); string transferId = GetNewTransferId(); string sourcePath = "https://storageaccount.blob.core.windows.net/container/blobsource"; string destinationPath = "https://storageaccount.blob.core.windows.net/container/blobdest"; @@ -132,12 +114,9 @@ public async Task RehydrateBlockBlob( StorageResourceType destinationType = StorageResourceType.BlockBlob; DataTransferProperties transferProperties = GetProperties( - test.DirectoryPath, transferId, sourcePath, destinationPath, - ToResourceId(sourceType), - ToResourceId(destinationType), ToProviderId(sourceType), ToProviderId(destinationType), isContainer: false, @@ -155,7 +134,6 @@ public async Task RehydrateBlockBlob( [Test] public async Task RehydrateBlockBlob_Options() { - using DisposingLocalDirectory test = DisposingLocalDirectory.GetTestDirectory(); string transferId = GetNewTransferId(); string sourcePath = "https://storageaccount.blob.core.windows.net/container/blobsource"; string destinationPath = "https://storageaccount.blob.core.windows.net/container/blobdest"; @@ -165,12 +143,9 @@ public async Task RehydrateBlockBlob_Options() BlobDestinationCheckpointData checkpointData = GetPopulatedDestinationCheckpointData(BlobType.Block, AccessTier.Cool); DataTransferProperties transferProperties = GetProperties( - test.DirectoryPath, transferId, sourcePath, destinationPath, - ToResourceId(sourceType), - ToResourceId(destinationType), ToProviderId(sourceType), ToProviderId(destinationType), isContainer: false, @@ -195,7 +170,6 @@ public async Task RehydrateBlockBlob_Options() public async Task RehydratePageBlob( [Values(true, false)] bool isSource) { - using DisposingLocalDirectory test = DisposingLocalDirectory.GetTestDirectory(); string transferId = GetNewTransferId(); string sourcePath = "https://storageaccount.blob.core.windows.net/container/blobsource"; string destinationPath = "https://storageaccount.blob.core.windows.net/container/blobdest"; @@ -205,12 +179,9 @@ public async Task RehydratePageBlob( StorageResourceType destinationType = StorageResourceType.PageBlob; DataTransferProperties transferProperties = GetProperties( - test.DirectoryPath, transferId, sourcePath, destinationPath, - ToResourceId(sourceType), - ToResourceId(destinationType), ToProviderId(sourceType), ToProviderId(destinationType), isContainer: false, @@ -228,7 +199,6 @@ public async Task RehydratePageBlob( [Test] public async Task RehydratePageBlob_Options() { - using DisposingLocalDirectory test = DisposingLocalDirectory.GetTestDirectory(); string transferId = GetNewTransferId(); string sourcePath = "https://storageaccount.blob.core.windows.net/container/blobsource"; string destinationPath = "https://storageaccount.blob.core.windows.net/container/blobdest"; @@ -238,12 +208,9 @@ public async Task RehydratePageBlob_Options() BlobDestinationCheckpointData checkpointData = GetPopulatedDestinationCheckpointData(BlobType.Page, AccessTier.P30); DataTransferProperties transferProperties = GetProperties( - test.DirectoryPath, transferId, sourcePath, destinationPath, - ToResourceId(sourceType), - ToResourceId(destinationType), ToProviderId(sourceType), ToProviderId(destinationType), isContainer: false, @@ -268,7 +235,6 @@ public async Task RehydratePageBlob_Options() public async Task RehydrateAppendBlob( [Values(true, false)] bool isSource) { - using DisposingLocalDirectory test = DisposingLocalDirectory.GetTestDirectory(); string transferId = GetNewTransferId(); string sourcePath = "https://storageaccount.blob.core.windows.net/container/blobsource"; string destinationPath = "https://storageaccount.blob.core.windows.net/container/blobdest"; @@ -278,12 +244,9 @@ public async Task RehydrateAppendBlob( StorageResourceType destinationType = StorageResourceType.AppendBlob; DataTransferProperties transferProperties = GetProperties( - test.DirectoryPath, transferId, sourcePath, destinationPath, - ToResourceId(sourceType), - ToResourceId(destinationType), ToProviderId(sourceType), ToProviderId(destinationType), isContainer: false, @@ -301,7 +264,6 @@ public async Task RehydrateAppendBlob( [Test] public async Task RehydrateAppendBlob_Options() { - using DisposingLocalDirectory test = DisposingLocalDirectory.GetTestDirectory(); string transferId = GetNewTransferId(); string sourcePath = "https://storageaccount.blob.core.windows.net/container/blobsource"; string destinationPath = "https://storageaccount.blob.core.windows.net/container/blobdest"; @@ -311,12 +273,9 @@ public async Task RehydrateAppendBlob_Options() BlobDestinationCheckpointData checkpointData = GetPopulatedDestinationCheckpointData(BlobType.Append, accessTier: default); DataTransferProperties transferProperties = GetProperties( - test.DirectoryPath, transferId, sourcePath, destinationPath, - ToResourceId(sourceType), - ToResourceId(destinationType), ToProviderId(sourceType), ToProviderId(destinationType), isContainer: false, @@ -341,7 +300,6 @@ public async Task RehydrateAppendBlob_Options() public async Task RehydrateBlobContainer( [Values(true, false)] bool isSource) { - using DisposingLocalDirectory test = DisposingLocalDirectory.GetTestDirectory(); string transferId = GetNewTransferId(); List sourcePaths = new List(); string sourceParentPath = "https://storageaccount.blob.core.windows.net/sourcecontainer"; @@ -361,12 +319,9 @@ public async Task RehydrateBlobContainer( string originalPath = isSource ? sourceParentPath : destinationParentPath; DataTransferProperties transferProperties = GetProperties( - test.DirectoryPath, transferId, sourceParentPath, destinationParentPath, - ToResourceId(sourceType), - ToResourceId(destinationType), ToProviderId(sourceType), ToProviderId(destinationType), isContainer: true, diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryStorageResourceContainer.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryStorageResourceContainer.cs index 9f3a44265bfd..863839eeda4a 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryStorageResourceContainer.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryStorageResourceContainer.cs @@ -50,12 +50,12 @@ protected override async IAsyncEnumerable GetStorageResourcesAs } } - public override StorageResourceCheckpointData GetSourceCheckpointData() + protected override StorageResourceCheckpointData GetSourceCheckpointData() { return new ShareFileSourceCheckpointData(); } - public override StorageResourceCheckpointData GetDestinationCheckpointData() + protected override StorageResourceCheckpointData GetDestinationCheckpointData() { return new ShareFileDestinationCheckpointData(); } diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileDestinationCheckpointData.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileDestinationCheckpointData.cs index 2a1624015970..da27081b0b78 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileDestinationCheckpointData.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileDestinationCheckpointData.cs @@ -9,7 +9,7 @@ internal class ShareFileDestinationCheckpointData : StorageResourceCheckpointDat { public override int Length => 0; - public override void Serialize(Stream stream) + protected override void Serialize(Stream stream) { } } diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileSourceCheckpointData.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileSourceCheckpointData.cs index daa9f96a0e8f..9b0bc38010ad 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileSourceCheckpointData.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileSourceCheckpointData.cs @@ -9,7 +9,7 @@ internal class ShareFileSourceCheckpointData : StorageResourceCheckpointData { public override int Length => 0; - public override void Serialize(Stream stream) + protected override void Serialize(Stream stream) { } } diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs index 54be94be43a6..795370f50ead 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs @@ -200,12 +200,12 @@ protected override async Task ReadStreamAsync( return response.Value.ToStorageResourceReadStreamResult(); } - public override StorageResourceCheckpointData GetSourceCheckpointData() + protected override StorageResourceCheckpointData GetSourceCheckpointData() { return new ShareFileSourceCheckpointData(); } - public override StorageResourceCheckpointData GetDestinationCheckpointData() + protected override StorageResourceCheckpointData GetDestinationCheckpointData() { return new ShareFileDestinationCheckpointData(); } diff --git a/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.net6.0.cs b/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.net6.0.cs index 3e2cd2474113..4140218d0bdb 100644 --- a/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.net6.0.cs +++ b/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.net6.0.cs @@ -62,15 +62,12 @@ protected internal DataTransferProgress() { } public partial class DataTransferProperties { protected internal DataTransferProperties() { } - public virtual Azure.Storage.DataMovement.TransferCheckpointStoreOptions Checkpointer { get { throw null; } } public virtual byte[] DestinationCheckpointData { get { throw null; } } public virtual string DestinationProviderId { get { throw null; } } - public virtual string DestinationTypeId { get { throw null; } } public virtual System.Uri DestinationUri { get { throw null; } } public virtual bool IsContainer { get { throw null; } } public virtual byte[] SourceCheckpointData { get { throw null; } } public virtual string SourceProviderId { get { throw null; } } - public virtual string SourceTypeId { get { throw null; } } public virtual System.Uri SourceUri { get { throw null; } } public virtual string TransferId { get { throw null; } } } @@ -119,14 +116,14 @@ protected StorageResource() { } protected internal abstract bool IsContainer { get; } public abstract string ProviderId { get; } public abstract System.Uri Uri { get; } - public abstract Azure.Storage.DataMovement.StorageResourceCheckpointData GetDestinationCheckpointData(); - public abstract Azure.Storage.DataMovement.StorageResourceCheckpointData GetSourceCheckpointData(); + protected internal abstract Azure.Storage.DataMovement.StorageResourceCheckpointData GetDestinationCheckpointData(); + protected internal abstract Azure.Storage.DataMovement.StorageResourceCheckpointData GetSourceCheckpointData(); } public abstract partial class StorageResourceCheckpointData { protected StorageResourceCheckpointData() { } public abstract int Length { get; } - public abstract void Serialize(System.IO.Stream stream); + protected internal abstract void Serialize(System.IO.Stream stream); } public abstract partial class StorageResourceContainer : Azure.Storage.DataMovement.StorageResource { diff --git a/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.netstandard2.0.cs b/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.netstandard2.0.cs index 3e2cd2474113..4140218d0bdb 100644 --- a/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.netstandard2.0.cs @@ -62,15 +62,12 @@ protected internal DataTransferProgress() { } public partial class DataTransferProperties { protected internal DataTransferProperties() { } - public virtual Azure.Storage.DataMovement.TransferCheckpointStoreOptions Checkpointer { get { throw null; } } public virtual byte[] DestinationCheckpointData { get { throw null; } } public virtual string DestinationProviderId { get { throw null; } } - public virtual string DestinationTypeId { get { throw null; } } public virtual System.Uri DestinationUri { get { throw null; } } public virtual bool IsContainer { get { throw null; } } public virtual byte[] SourceCheckpointData { get { throw null; } } public virtual string SourceProviderId { get { throw null; } } - public virtual string SourceTypeId { get { throw null; } } public virtual System.Uri SourceUri { get { throw null; } } public virtual string TransferId { get { throw null; } } } @@ -119,14 +116,14 @@ protected StorageResource() { } protected internal abstract bool IsContainer { get; } public abstract string ProviderId { get; } public abstract System.Uri Uri { get; } - public abstract Azure.Storage.DataMovement.StorageResourceCheckpointData GetDestinationCheckpointData(); - public abstract Azure.Storage.DataMovement.StorageResourceCheckpointData GetSourceCheckpointData(); + protected internal abstract Azure.Storage.DataMovement.StorageResourceCheckpointData GetDestinationCheckpointData(); + protected internal abstract Azure.Storage.DataMovement.StorageResourceCheckpointData GetSourceCheckpointData(); } public abstract partial class StorageResourceCheckpointData { protected StorageResourceCheckpointData() { } public abstract int Length { get; } - public abstract void Serialize(System.IO.Stream stream); + protected internal abstract void Serialize(System.IO.Stream stream); } public abstract partial class StorageResourceContainer : Azure.Storage.DataMovement.StorageResource { diff --git a/sdk/storage/Azure.Storage.DataMovement/src/CheckpointerExtensions.cs b/sdk/storage/Azure.Storage.DataMovement/src/CheckpointerExtensions.cs index 965e9620e79a..2b411bb8357b 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/CheckpointerExtensions.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/CheckpointerExtensions.cs @@ -9,8 +9,23 @@ namespace Azure.Storage.DataMovement { - internal partial class CheckpointerExtensions + internal static partial class CheckpointerExtensions { + internal static TransferCheckpointer GetCheckpointer(this TransferCheckpointStoreOptions options) + { + if (!string.IsNullOrEmpty(options?.CheckpointerPath)) + { + return new LocalTransferCheckpointer(options.CheckpointerPath); + } + else + { + // Default TransferCheckpointer + return new LocalTransferCheckpointer(default); + } + } + + internal static bool IsLocalResource(this StorageResource resource) => resource.Uri.IsFile; + internal static async Task GetJobStatusAsync( this TransferCheckpointer checkpointer, string transferId, @@ -54,24 +69,12 @@ internal static async Task GetDataTransferPropertiesAsyn header = JobPlanHeader.Deserialize(stream); } - string sourceTypeId = default; - string destinationTypeId = default; - // Only need to get type ids for single transfers - if (!header.IsContainer) - { - (sourceTypeId, destinationTypeId) = await checkpointer.GetResourceIdsAsync( - transferId, - cancellationToken).ConfigureAwait(false); - } - return new DataTransferProperties { TransferId = transferId, - SourceTypeId = sourceTypeId, SourceUri = new Uri(header.ParentSourcePath), SourceProviderId = header.SourceProviderId, SourceCheckpointData = header.SourceCheckpointData, - DestinationTypeId = destinationTypeId, DestinationUri = new Uri(header.ParentDestinationPath), DestinationProviderId = header.DestinationProviderId, DestinationCheckpointData = header.DestinationCheckpointData, diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataTransfer.cs b/sdk/storage/Azure.Storage.DataMovement/src/DataTransfer.cs similarity index 100% rename from sdk/storage/Azure.Storage.DataMovement/src/Shared/DataTransfer.cs rename to sdk/storage/Azure.Storage.DataMovement/src/DataTransfer.cs diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataTransferInternalState.cs b/sdk/storage/Azure.Storage.DataMovement/src/DataTransferInternalState.cs similarity index 100% rename from sdk/storage/Azure.Storage.DataMovement/src/Shared/DataTransferInternalState.cs rename to sdk/storage/Azure.Storage.DataMovement/src/DataTransferInternalState.cs diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataTransferOrder.cs b/sdk/storage/Azure.Storage.DataMovement/src/DataTransferOrder.cs similarity index 100% rename from sdk/storage/Azure.Storage.DataMovement/src/Shared/DataTransferOrder.cs rename to sdk/storage/Azure.Storage.DataMovement/src/DataTransferOrder.cs diff --git a/sdk/storage/Azure.Storage.DataMovement/src/DataTransferProperties.cs b/sdk/storage/Azure.Storage.DataMovement/src/DataTransferProperties.cs index 6ddf75526257..790d997fae84 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/DataTransferProperties.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/DataTransferProperties.cs @@ -6,26 +6,15 @@ namespace Azure.Storage.DataMovement { /// - /// Storage Resource Item properties returned by + /// Properties of a transfer that can be used for resuming. /// public class DataTransferProperties { - /// - /// Contains the checkpointer information to rehydrate the StorageResource from. - /// - public virtual TransferCheckpointStoreOptions Checkpointer { get; internal set; } - /// /// Contains the transfer ID which to rehydrate the StorageResource from. /// public virtual string TransferId { get; internal set; } - /// - /// Contains the type id for the source resource to use during rehydration. - /// Will be null if is true. - /// - public virtual string SourceTypeId { get; internal set; } - /// /// Contains the Source uri of the Storage Resource. /// @@ -41,12 +30,6 @@ public class DataTransferProperties /// public virtual byte[] SourceCheckpointData { get; internal set; } - /// - /// Contains the type id for the destination resource to use during rehydration. - /// Will be null if is true. - /// - public virtual string DestinationTypeId { get; internal set; } - /// /// Contains the Destination uri of the Storage Resource. /// diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/FolderPropertiesMode.cs b/sdk/storage/Azure.Storage.DataMovement/src/JobPlan/FolderPropertiesMode.cs similarity index 100% rename from sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/FolderPropertiesMode.cs rename to sdk/storage/Azure.Storage.DataMovement/src/JobPlan/FolderPropertiesMode.cs diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobChunkPlanBody.cs b/sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobChunkPlanBody.cs similarity index 100% rename from sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobChunkPlanBody.cs rename to sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobChunkPlanBody.cs diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPartDeleteSnapshotsOption.cs b/sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPartDeleteSnapshotsOption.cs similarity index 100% rename from sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPartDeleteSnapshotsOption.cs rename to sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPartDeleteSnapshotsOption.cs diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPartPermanentDeleteOption.cs b/sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPartPermanentDeleteOption.cs similarity index 100% rename from sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPartPermanentDeleteOption.cs rename to sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPartPermanentDeleteOption.cs diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPartPlanBlockBlobTier.cs b/sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPartPlanBlockBlobTier.cs similarity index 100% rename from sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPartPlanBlockBlobTier.cs rename to sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPartPlanBlockBlobTier.cs diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPartPlanDestinationBlob.cs b/sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPartPlanDestinationBlob.cs similarity index 100% rename from sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPartPlanDestinationBlob.cs rename to sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPartPlanDestinationBlob.cs diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPartPlanDestinationLocal.cs b/sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPartPlanDestinationLocal.cs similarity index 100% rename from sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPartPlanDestinationLocal.cs rename to sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPartPlanDestinationLocal.cs diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPartPlanFile.cs b/sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPartPlanFile.cs similarity index 100% rename from sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPartPlanFile.cs rename to sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPartPlanFile.cs diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPartPlanFileName.cs b/sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPartPlanFileName.cs similarity index 100% rename from sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPartPlanFileName.cs rename to sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPartPlanFileName.cs diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPartPlanHeader.cs b/sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPartPlanHeader.cs similarity index 99% rename from sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPartPlanHeader.cs rename to sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPartPlanHeader.cs index b7790c40a731..20b7e7b7a837 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPartPlanHeader.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPartPlanHeader.cs @@ -852,12 +852,12 @@ public static JobPartPlanHeader Deserialize(Stream stream) // AtomicPartStatus.HasSkippedItems bool partStatusHasSkipped = Convert.ToBoolean(reader.ReadByte()); - DataTransferStatus atomicJobStatus = new DataTransferStatusInternal( + DataTransferStatus atomicJobStatus = new DataTransferStatus( jobStatusState, jobStatusHasFailed, jobStatusHasSkipped); - DataTransferStatus atomicPartStatus = new DataTransferStatusInternal( + DataTransferStatus atomicPartStatus = new DataTransferStatus( partStatusState, partStatusHasFailed, partStatusHasSkipped); diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPartPlanPageBlobTier.cs b/sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPartPlanPageBlobTier.cs similarity index 100% rename from sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPartPlanPageBlobTier.cs rename to sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPartPlanPageBlobTier.cs diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPartPlanRehydratePriorityType.cs b/sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPartPlanRehydratePriorityType.cs similarity index 100% rename from sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPartPlanRehydratePriorityType.cs rename to sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPartPlanRehydratePriorityType.cs diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPlanBlobType.cs b/sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPlanBlobType.cs similarity index 100% rename from sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPlanBlobType.cs rename to sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPlanBlobType.cs diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPlanFile.cs b/sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPlanFile.cs similarity index 100% rename from sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPlanFile.cs rename to sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPlanFile.cs diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPlanHeader.cs b/sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPlanHeader.cs similarity index 96% rename from sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPlanHeader.cs rename to sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPlanHeader.cs index c8433d652b68..5d1615ccdc80 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPlanHeader.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPlanHeader.cs @@ -197,17 +197,17 @@ public void Serialize(Stream stream) // ParentSourcePath offset/length byte[] parentSourcePathBytes = Encoding.UTF8.GetBytes(ParentSourcePath); - JobPlanExtensions.WriteVariableLengthFieldInfo(writer, parentSourcePathBytes.Length, ref currentVariableLengthIndex); + writer.WriteVariableLengthFieldInfo(parentSourcePathBytes.Length, ref currentVariableLengthIndex); // ParentDestinationPath offset/length byte[] parentDestinationPathBytes = Encoding.UTF8.GetBytes(ParentDestinationPath); - JobPlanExtensions.WriteVariableLengthFieldInfo(writer, parentDestinationPathBytes.Length, ref currentVariableLengthIndex); + writer.WriteVariableLengthFieldInfo(parentDestinationPathBytes.Length, ref currentVariableLengthIndex); // SourceCheckpointData offset/length - JobPlanExtensions.WriteVariableLengthFieldInfo(writer, _sourceCheckpointData.Length, ref currentVariableLengthIndex); + writer.WriteVariableLengthFieldInfo(_sourceCheckpointData.Length, ref currentVariableLengthIndex); // DestinationCheckpointData offset/length - JobPlanExtensions.WriteVariableLengthFieldInfo(writer, _destinationCheckpointData.Length, ref currentVariableLengthIndex); + writer.WriteVariableLengthFieldInfo(_destinationCheckpointData.Length, ref currentVariableLengthIndex); // ParentSourcePath writer.Write(parentSourcePathBytes); diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPlanOperation.cs b/sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPlanOperation.cs similarity index 100% rename from sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPlanOperation.cs rename to sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPlanOperation.cs diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPlanStatus.cs b/sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPlanStatus.cs similarity index 100% rename from sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPlanStatus.cs rename to sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPlanStatus.cs diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlanExtensions.cs b/sdk/storage/Azure.Storage.DataMovement/src/JobPlanExtensions.cs similarity index 54% rename from sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlanExtensions.cs rename to sdk/storage/Azure.Storage.DataMovement/src/JobPlanExtensions.cs index c0d41f039510..d8368553b2ab 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlanExtensions.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/JobPlanExtensions.cs @@ -2,10 +2,8 @@ // Licensed under the MIT License. using System; -using System.Collections.Generic; using System.IO; using System.IO.MemoryMappedFiles; -using System.Text; using System.Threading; using System.Threading.Tasks; using Azure.Storage.DataMovement.JobPlan; @@ -14,21 +12,6 @@ namespace Azure.Storage.DataMovement { internal static partial class JobPlanExtensions { - public static string ToString(this byte[] bytes, long length) - { - return Encoding.UTF8.GetString(bytes, 0, (int)length); - } - - public static long ToLong(this byte[] bytes) - { - return BitConverter.ToInt64(bytes, 0); - } - - public static ushort ToUShort(this byte[] bytes) - { - return BitConverter.ToUInt16(bytes, 0); - } - internal static JobPartPlanHeader GetJobPartPlanHeader(this JobPartPlanFileName fileName) { JobPartPlanHeader result; @@ -46,78 +29,6 @@ internal static JobPartPlanHeader GetJobPartPlanHeader(this JobPartPlanFileName return result; } - internal static async Task<(string Source, string Destination)> GetResourceIdsAsync( - this TransferCheckpointer checkpointer, - string transferId, - CancellationToken cancellationToken) - { - int startIndex = DataMovementConstants.JobPartPlanFile.SourceResourceIdLengthIndex; - int readLength = DataMovementConstants.JobPartPlanFile.DestinationPathLengthIndex - startIndex; - - string sourceResourceId; - string destinationResourceId; - using (Stream stream = await checkpointer.ReadJobPartPlanFileAsync( - transferId: transferId, - partNumber: 0, - offset: startIndex, - length: readLength, - cancellationToken: cancellationToken).ConfigureAwait(false)) - { - BinaryReader reader = new BinaryReader(stream); - - // Read Source Length - byte[] sourceLengthBuffer = reader.ReadBytes(DataMovementConstants.UShortSizeInBytes); - ushort sourceLength = sourceLengthBuffer.ToUShort(); - - // Read Source - byte[] sourceBuffer = reader.ReadBytes(DataMovementConstants.JobPartPlanFile.ResourceIdNumBytes); - sourceResourceId = sourceBuffer.ToString(sourceLength); - - // Set the stream position to the start of the destination resource id - reader.BaseStream.Position = DataMovementConstants.JobPartPlanFile.DestinationResourceIdLengthIndex - startIndex; - - // Read Destination Length - byte[] destLengthBuffer = reader.ReadBytes(DataMovementConstants.UShortSizeInBytes); - ushort destLength = destLengthBuffer.ToUShort(); - - // Read Destination - byte[] destBuffer = reader.ReadBytes(DataMovementConstants.JobPartPlanFile.ResourceIdNumBytes); - destinationResourceId = destBuffer.ToString(destLength); - } - - return (sourceResourceId, destinationResourceId); - } - - internal static IDictionary ToDictionary(this string str, string elementName) - { - IDictionary dictionary = new Dictionary(); - string[] splitSemiColon = str.Split(';'); - foreach (string value in splitSemiColon) - { - if (!string.IsNullOrEmpty(value)) - { - string[] splitEqual = value.Split('='); - if (splitEqual.Length != 2) - { - throw Errors.InvalidStringToDictionary(elementName, str); - } - dictionary.Add(splitEqual[0], splitEqual[1]); - } - } - return dictionary; - } - - internal static string DictionaryToString(this IDictionary dict) - { - string concatStr = ""; - foreach (KeyValuePair kv in dict) - { - // e.g. store like "header=value;" - concatStr = string.Concat(concatStr, $"{kv.Key}={kv.Value};"); - } - return concatStr; - } - internal static async Task GetHeaderUShortValue( this TransferCheckpointer checkpointer, string transferId, @@ -253,45 +164,7 @@ internal static DataTransferStatus ToDataTransferStatus(this JobPlanStatus jobPl bool hasFailed = jobPlanStatus.HasFlag(JobPlanStatus.HasFailed); bool hasSkipped = jobPlanStatus.HasFlag(JobPlanStatus.HasSkipped); - return new DataTransferStatusInternal(state, hasFailed, hasSkipped); - } - - /// - /// Writes the given length and offset and increments currentOffset accordingly. - /// - /// The writer to write to. - /// The length of the variable length field. - /// - /// A reference to the current offset of the variable length fields - /// that will be used to set the offset and then incremented. - /// - internal static void WriteVariableLengthFieldInfo( - BinaryWriter writer, - int length, - ref int currentOffset) - { - // Write the offset, -1 if size is 0 - if (length > 0) - { - writer.Write(currentOffset); - currentOffset += length; - } - else - { - writer.Write(-1); - } - - // Write the length - writer.Write(length); - } - - internal static string ToSanitizedString(this Uri uri) - { - UriBuilder builder = new(uri); - - // Remove any query parameters (including SAS) - builder.Query = string.Empty; - return builder.Uri.AbsoluteUri; + return new DataTransferStatus(state, hasFailed, hasSkipped); } } } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/LocalDestinationCheckpointData.cs b/sdk/storage/Azure.Storage.DataMovement/src/LocalDestinationCheckpointData.cs index d592b5faacea..97a45b05a20a 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/LocalDestinationCheckpointData.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/LocalDestinationCheckpointData.cs @@ -9,7 +9,7 @@ internal class LocalDestinationCheckpointData : StorageResourceCheckpointData { public override int Length => 0; - public override void Serialize(Stream stream) + protected internal override void Serialize(Stream stream) { } } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/LocalDirectoryStorageResourceContainer.cs b/sdk/storage/Azure.Storage.DataMovement/src/LocalDirectoryStorageResourceContainer.cs index a919df766810..8b0383e1f820 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/LocalDirectoryStorageResourceContainer.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/LocalDirectoryStorageResourceContainer.cs @@ -80,12 +80,12 @@ protected internal override async IAsyncEnumerable GetStorageRe } } - public override StorageResourceCheckpointData GetSourceCheckpointData() + protected internal override StorageResourceCheckpointData GetSourceCheckpointData() { return new LocalSourceCheckpointData(); } - public override StorageResourceCheckpointData GetDestinationCheckpointData() + protected internal override StorageResourceCheckpointData GetDestinationCheckpointData() { return new LocalDestinationCheckpointData(); } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/LocalFileStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement/src/LocalFileStorageResource.cs index cdb84b6fb300..52c9fe4f5f59 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/LocalFileStorageResource.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/LocalFileStorageResource.cs @@ -276,12 +276,12 @@ protected internal override Task DeleteIfExistsAsync(CancellationToken can return Task.FromResult(false); } - public override StorageResourceCheckpointData GetSourceCheckpointData() + protected internal override StorageResourceCheckpointData GetSourceCheckpointData() { return new LocalSourceCheckpointData(); } - public override StorageResourceCheckpointData GetDestinationCheckpointData() + protected internal override StorageResourceCheckpointData GetDestinationCheckpointData() { return new LocalDestinationCheckpointData(); } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/LocalSourceCheckpointData.cs b/sdk/storage/Azure.Storage.DataMovement/src/LocalSourceCheckpointData.cs index d4aa8a90877b..48e09453d049 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/LocalSourceCheckpointData.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/LocalSourceCheckpointData.cs @@ -9,7 +9,7 @@ internal class LocalSourceCheckpointData : StorageResourceCheckpointData { public override int Length => 0; - public override void Serialize(Stream stream) + protected internal override void Serialize(Stream stream) { } } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/LocalTransferCheckpointer.cs b/sdk/storage/Azure.Storage.DataMovement/src/LocalTransferCheckpointer.cs similarity index 98% rename from sdk/storage/Azure.Storage.DataMovement/src/Shared/LocalTransferCheckpointer.cs rename to sdk/storage/Azure.Storage.DataMovement/src/LocalTransferCheckpointer.cs index a2dbca82d5ac..b36b131725f6 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/Shared/LocalTransferCheckpointer.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/LocalTransferCheckpointer.cs @@ -55,7 +55,6 @@ public LocalTransferCheckpointer(string folderPath) } } - /// public override async Task AddNewJobAsync( string transferId, StorageResource source, @@ -77,7 +76,7 @@ public override async Task AddNewJobAsync( destination.ProviderId, isContainer, false, /* enumerationComplete */ - new DataTransferStatusInternal(), + new DataTransferStatus(), source.Uri.ToSanitizedString(), destination.Uri.ToSanitizedString(), source.GetSourceCheckpointData(), @@ -95,7 +94,6 @@ public override async Task AddNewJobAsync( } } - /// public override async Task AddNewJobPartAsync( string transferId, int partNumber, @@ -128,7 +126,6 @@ public override async Task AddNewJobPartAsync( } } - /// public override Task CurrentJobPartCountAsync( string transferId, CancellationToken cancellationToken = default) @@ -141,7 +138,6 @@ public override Task CurrentJobPartCountAsync( throw Errors.MissingTransferIdCheckpointer(transferId); } - /// public override async Task ReadJobPlanFileAsync( string transferId, int offset, @@ -174,7 +170,6 @@ public override async Task ReadJobPlanFileAsync( } } - /// public override async Task ReadJobPartPlanFileAsync( string transferId, int partNumber, @@ -215,7 +210,6 @@ public override async Task ReadJobPartPlanFileAsync( } } - /// public override async Task WriteToJobPlanFileAsync( string transferId, int fileOffset, @@ -246,7 +240,6 @@ public override async Task WriteToJobPlanFileAsync( } } - /// public override Task TryRemoveStoredTransferAsync(string transferId, CancellationToken cancellationToken = default) { Argument.AssertNotNullOrWhiteSpace(transferId, nameof(transferId)); @@ -293,13 +286,11 @@ public override Task TryRemoveStoredTransferAsync(string transferId, Cance return Task.FromResult(result); } - /// public override Task> GetStoredTransfersAsync(CancellationToken cancellationToken = default) { return Task.FromResult(_transferStates.Keys.ToList()); } - /// public override async Task SetJobTransferStatusAsync( string transferId, DataTransferStatus status, @@ -331,7 +322,6 @@ public override async Task SetJobTransferStatusAsync( } } - /// public override async Task SetJobPartTransferStatusAsync( string transferId, int partNumber, diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/CheckpointerExtensions.cs b/sdk/storage/Azure.Storage.DataMovement/src/Shared/CheckpointerExtensions.cs index 5101526b1e55..3424079970b1 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/Shared/CheckpointerExtensions.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/Shared/CheckpointerExtensions.cs @@ -1,23 +1,90 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + namespace Azure.Storage.DataMovement { internal static partial class CheckpointerExtensions { - public static TransferCheckpointer GetCheckpointer(this TransferCheckpointStoreOptions options) + public static string ToString(this byte[] bytes, long length) + { + return Encoding.UTF8.GetString(bytes, 0, (int)length); + } + + public static long ToLong(this byte[] bytes) + { + return BitConverter.ToInt64(bytes, 0); + } + + public static ushort ToUShort(this byte[] bytes) + { + return BitConverter.ToUInt16(bytes, 0); + } + + internal static IDictionary ToDictionary(this string str, string elementName) + { + IDictionary dictionary = new Dictionary(); + string[] splitSemiColon = str.Split(';'); + foreach (string value in splitSemiColon) + { + if (!string.IsNullOrEmpty(value)) + { + string[] splitEqual = value.Split('='); + if (splitEqual.Length != 2) + { + throw Errors.InvalidStringToDictionary(elementName, str); + } + dictionary.Add(splitEqual[0], splitEqual[1]); + } + } + return dictionary; + } + + internal static string DictionaryToString(this IDictionary dict) + { + string concatStr = ""; + foreach (KeyValuePair kv in dict) + { + // e.g. store like "header=value;" + concatStr = string.Concat(concatStr, $"{kv.Key}={kv.Value};"); + } + return concatStr; + } + + /// + /// Writes the given length and offset and increments currentOffset accordingly. + /// + internal static void WriteVariableLengthFieldInfo( + this BinaryWriter writer, + int length, + ref int currentOffset) { - if (!string.IsNullOrEmpty(options?.CheckpointerPath)) + // Write the offset, -1 if size is 0 + if (length > 0) { - return new LocalTransferCheckpointer(options.CheckpointerPath); + writer.Write(currentOffset); + currentOffset += length; } else { - // Default TransferCheckpointer - return new LocalTransferCheckpointer(default); + writer.Write(-1); } + + // Write the length + writer.Write(length); } - internal static bool IsLocalResource(this StorageResource resource) => resource.Uri.IsFile; + internal static string ToSanitizedString(this Uri uri) + { + UriBuilder builder = new(uri); + + // Remove any query parameters (including SAS) + builder.Query = string.Empty; + return builder.Uri.AbsoluteUri; + } } } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/StorageResource.cs b/sdk/storage/Azure.Storage.DataMovement/src/StorageResource.cs index d120c9938375..7d5e092b3481 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/StorageResource.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/StorageResource.cs @@ -37,12 +37,12 @@ protected StorageResource() /// Gets the source checkpoint data for this resource that will be written to the checkpointer. /// /// A containing the checkpoint information for this resource. - public abstract StorageResourceCheckpointData GetSourceCheckpointData(); + protected internal abstract StorageResourceCheckpointData GetSourceCheckpointData(); /// /// Gets the destination checkpoint data for this resource that will be written to the checkpointer. /// /// A containing the checkpoint information for this resource. - public abstract StorageResourceCheckpointData GetDestinationCheckpointData(); + protected internal abstract StorageResourceCheckpointData GetDestinationCheckpointData(); } } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceCheckpointData.cs b/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceCheckpointData.cs index d47613fe417e..fa23a33e53bc 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceCheckpointData.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceCheckpointData.cs @@ -19,6 +19,6 @@ public abstract class StorageResourceCheckpointData /// Serializes the checkpoint data into the given stream. /// /// The stream to serialize the data into. - public abstract void Serialize(Stream stream); + protected internal abstract void Serialize(Stream stream); } } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/StorageResourceCreationPreference.cs b/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceCreationPreference.cs similarity index 100% rename from sdk/storage/Azure.Storage.DataMovement/src/Shared/StorageResourceCreationPreference.cs rename to sdk/storage/Azure.Storage.DataMovement/src/StorageResourceCreationPreference.cs diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/TransferCheckpointer.cs b/sdk/storage/Azure.Storage.DataMovement/src/TransferCheckpointer.cs similarity index 100% rename from sdk/storage/Azure.Storage.DataMovement/src/Shared/TransferCheckpointer.cs rename to sdk/storage/Azure.Storage.DataMovement/src/TransferCheckpointer.cs diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/TransferManager.cs b/sdk/storage/Azure.Storage.DataMovement/src/TransferManager.cs similarity index 99% rename from sdk/storage/Azure.Storage.DataMovement/src/Shared/TransferManager.cs rename to sdk/storage/Azure.Storage.DataMovement/src/TransferManager.cs index 94d625a21ea6..a87db06f89af 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/Shared/TransferManager.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/TransferManager.cs @@ -304,7 +304,6 @@ public virtual async IAsyncEnumerable GetResumableTransf DataTransferProperties properties = await _checkpointer.GetDataTransferPropertiesAsync( transferId, _cancellationToken).ConfigureAwait(false); - properties.Checkpointer = _checkpointerOptions; yield return properties; } } @@ -357,7 +356,6 @@ public virtual async Task ResumeTransferAsync( DataTransferProperties properties = await _checkpointer.GetDataTransferPropertiesAsync( transferId, cancellationToken).ConfigureAwait(false); - properties.Checkpointer = _checkpointerOptions; return await ResumeTransferAsync(properties, transferOptions, cancellationToken).ConfigureAwait(false); } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/TransferStatusEventArgs.cs b/sdk/storage/Azure.Storage.DataMovement/src/TransferStatusEventArgs.cs similarity index 100% rename from sdk/storage/Azure.Storage.DataMovement/src/Shared/TransferStatusEventArgs.cs rename to sdk/storage/Azure.Storage.DataMovement/src/TransferStatusEventArgs.cs diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/CheckpointerTesting.cs b/sdk/storage/Azure.Storage.DataMovement/tests/CheckpointerTesting.cs similarity index 97% rename from sdk/storage/Azure.Storage.DataMovement/tests/Shared/CheckpointerTesting.cs rename to sdk/storage/Azure.Storage.DataMovement/tests/CheckpointerTesting.cs index 538a75d48cca..2947248c6c5d 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/CheckpointerTesting.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/CheckpointerTesting.cs @@ -1,18 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -extern alias DMBlobs; - +using Azure.Storage.DataMovement.JobPlan; using Azure.Storage.Test; using System.Collections.Generic; using System.IO; using System.Threading.Tasks; using System; -#if BlobDataMovementSDK -using DMBlobs::Azure.Storage.DataMovement.JobPlan; -#else -using Azure.Storage.DataMovement.JobPlan; -#endif using NUnit.Framework; namespace Azure.Storage.DataMovement.Tests @@ -56,8 +50,8 @@ internal static readonly DateTimeOffset DefaultStartTime internal const JobPartDeleteSnapshotsOption DefaultDeleteSnapshotsOption = JobPartDeleteSnapshotsOption.None; internal const JobPartPermanentDeleteOption DefaultPermanentDeleteOption = JobPartPermanentDeleteOption.None; internal const JobPartPlanRehydratePriorityType DefaultRehydratePriorityType = JobPartPlanRehydratePriorityType.None; - internal static readonly DataTransferStatus DefaultJobStatus = new DataTransferStatusInternal(DataTransferState.Queued, false, false); - internal static readonly DataTransferStatus DefaultPartStatus = new DataTransferStatusInternal(DataTransferState.Queued, false, false); + internal static readonly DataTransferStatus DefaultJobStatus = new DataTransferStatus(DataTransferState.Queued, false, false); + internal static readonly DataTransferStatus DefaultPartStatus = new DataTransferStatus(DataTransferState.Queued, false, false); internal static readonly DateTimeOffset DefaultCreateTime = new DateTimeOffset(2023, 08, 28, 17, 26, 0, default); internal static JobPartPlanHeader CreateDefaultJobPartHeader( diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/GetTransfersTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/GetTransfersTests.cs index a1f85c4887a7..8d43cd2f2f4e 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/GetTransfersTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/GetTransfersTests.cs @@ -242,11 +242,11 @@ public async Task GetResumableTransfers_LocalCheckpointer() // Build expected results first to use to populate checkpointer DataTransferProperties[] expectedResults = new DataTransferProperties[] { - new DataTransferProperties { TransferId = Guid.NewGuid().ToString(), SourceProviderId = "local", SourceTypeId = "LocalFile", SourceUri = new Uri(parentLocalUri1, "file1"), DestinationProviderId = "blob", DestinationTypeId = "BlockBlob", DestinationUri = new Uri(parentRemoteUri, "file1"), IsContainer = false }, - new DataTransferProperties { TransferId = Guid.NewGuid().ToString(), SourceProviderId = "blob", SourceTypeId = "BlockBlob", SourceUri = new Uri(parentRemoteUri, "file2/"), DestinationProviderId = "local", DestinationTypeId = "LocalFile", DestinationUri = new Uri(parentLocalUri1, "file2/"), IsContainer = false }, - new DataTransferProperties { TransferId = Guid.NewGuid().ToString(), SourceProviderId = "blob", SourceTypeId = "BlockBlob", SourceUri = new Uri(parentRemoteUri, "file3"), DestinationProviderId = "blob", DestinationTypeId = "BlockBlob", DestinationUri = new Uri(parentRemoteUri, "file3"), IsContainer = false }, - new DataTransferProperties { TransferId = Guid.NewGuid().ToString(), SourceProviderId = "blob", SourceTypeId = default, SourceUri = parentRemoteUri, DestinationProviderId = "local", DestinationTypeId = default, DestinationUri = parentLocalUri1, IsContainer = true }, - new DataTransferProperties { TransferId = Guid.NewGuid().ToString(), SourceProviderId = "local", SourceTypeId = default, SourceUri = parentLocalUri2, DestinationProviderId = "blob", DestinationTypeId = default, DestinationUri = parentRemoteUri, IsContainer = true }, + new DataTransferProperties { TransferId = Guid.NewGuid().ToString(), SourceProviderId = "local", SourceUri = new Uri(parentLocalUri1, "file1"), DestinationProviderId = "blob", DestinationUri = new Uri(parentRemoteUri, "file1"), IsContainer = false }, + new DataTransferProperties { TransferId = Guid.NewGuid().ToString(), SourceProviderId = "blob", SourceUri = new Uri(parentRemoteUri, "file2/"), DestinationProviderId = "local", DestinationUri = new Uri(parentLocalUri1, "file2/"), IsContainer = false }, + new DataTransferProperties { TransferId = Guid.NewGuid().ToString(), SourceProviderId = "blob", SourceUri = new Uri(parentRemoteUri, "file3"), DestinationProviderId = "blob", DestinationUri = new Uri(parentRemoteUri, "file3"), IsContainer = false }, + new DataTransferProperties { TransferId = Guid.NewGuid().ToString(), SourceProviderId = "blob", SourceUri = parentRemoteUri, DestinationProviderId = "local", DestinationUri = parentLocalUri1, IsContainer = true }, + new DataTransferProperties { TransferId = Guid.NewGuid().ToString(), SourceProviderId = "local", SourceUri = parentLocalUri2, DestinationProviderId = "blob", DestinationUri = parentRemoteUri, IsContainer = true }, }; // Add a transfer for each expected result @@ -354,9 +354,7 @@ private void AddTransferFromDataTransferProperties( numParts, /* jobPartCount */ InProgressStatus, sourcePaths, - destinationPaths, - sourceResourceId: sourceTypeId, - destinationResourceId: destinationTypeId); + destinationPaths); } else { @@ -366,9 +364,7 @@ private void AddTransferFromDataTransferProperties( 1, /* jobPartCount */ InProgressStatus, new List { properties.SourceUri.AbsoluteUri }, - new List { properties.DestinationUri.AbsoluteUri }, - sourceResourceId: properties.SourceTypeId, - destinationResourceId: properties.DestinationTypeId); + new List { properties.DestinationUri.AbsoluteUri }); } } @@ -376,10 +372,8 @@ private void AssertTransferProperties(DataTransferProperties expected, DataTrans { Assert.AreEqual(expected.TransferId, actual.TransferId); Assert.AreEqual(expected.SourceProviderId, actual.SourceProviderId); - Assert.AreEqual(expected.SourceTypeId, actual.SourceTypeId); Assert.AreEqual(expected.SourceUri.AbsoluteUri.TrimEnd('\\', '/'), actual.SourceUri.AbsoluteUri.TrimEnd('\\', '/')); Assert.AreEqual(expected.DestinationProviderId, actual.DestinationProviderId); - Assert.AreEqual(expected.DestinationTypeId, actual.DestinationTypeId); Assert.AreEqual(expected.DestinationUri.AbsoluteUri.TrimEnd('\\', '/'), actual.DestinationUri.AbsoluteUri.TrimEnd('\\', '/')); Assert.AreEqual(expected.IsContainer, actual.IsContainer); diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/LocalTransferCheckpointerFactory.cs b/sdk/storage/Azure.Storage.DataMovement/tests/LocalTransferCheckpointerFactory.cs index 608a6f024bd1..8d513542bd19 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/LocalTransferCheckpointerFactory.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/LocalTransferCheckpointerFactory.cs @@ -45,8 +45,8 @@ internal static readonly DateTimeOffset _testStartTime internal const JobPartDeleteSnapshotsOption _testDeleteSnapshotsOption = JobPartDeleteSnapshotsOption.None; internal const JobPartPermanentDeleteOption _testPermanentDeleteOption = JobPartPermanentDeleteOption.None; internal const JobPartPlanRehydratePriorityType _testRehydratePriorityType = JobPartPlanRehydratePriorityType.None; - internal static readonly DataTransferStatus _testJobStatus = new DataTransferStatusInternal(DataTransferState.Queued, false, false); - internal static readonly DataTransferStatus _testPartStatus = new DataTransferStatusInternal(DataTransferState.Queued, false, false); + internal static readonly DataTransferStatus _testJobStatus = new DataTransferStatus(DataTransferState.Queued, false, false); + internal static readonly DataTransferStatus _testPartStatus = new DataTransferStatus(DataTransferState.Queued, false, false); private string _checkpointerPath; diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/MockResourceCheckpointData.cs b/sdk/storage/Azure.Storage.DataMovement/tests/MockResourceCheckpointData.cs index 855d7411db73..c205471a1fa6 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/MockResourceCheckpointData.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/MockResourceCheckpointData.cs @@ -32,7 +32,7 @@ public MockResourceCheckpointData() } } - public override void Serialize(Stream stream) + protected internal override void Serialize(Stream stream) { stream.Write(Bytes, 0, Bytes.Length); } diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/MockStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement/tests/MockStorageResource.cs index 2365d6782002..80bf379776f6 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/MockStorageResource.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/MockStorageResource.cs @@ -87,12 +87,12 @@ protected internal override Task ReadStreamAsyn return Task.FromResult(new StorageResourceReadStreamResult(_readStream)); } - public override StorageResourceCheckpointData GetSourceCheckpointData() + protected internal override StorageResourceCheckpointData GetSourceCheckpointData() { return new MockResourceCheckpointData(); } - public override StorageResourceCheckpointData GetDestinationCheckpointData() + protected internal override StorageResourceCheckpointData GetDestinationCheckpointData() { return new MockResourceCheckpointData(); } diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/RehydrateStorageResourceTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/RehydrateStorageResourceTests.cs index b003a84083cc..1d9e3654d685 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/RehydrateStorageResourceTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/RehydrateStorageResourceTests.cs @@ -6,7 +6,6 @@ using System.IO; using System.Threading; using System.Threading.Tasks; -using Azure.Storage.DataMovement.JobPlan; using Azure.Storage.Tests; using Moq; using NUnit.Framework; @@ -25,23 +24,10 @@ private enum StorageResourceType Local } - private static string ToResourceId(StorageResourceType type) - { - return type switch - { - StorageResourceType.BlockBlob => "BlockBlob", - StorageResourceType.Local => "LocalFile", - _ => throw new NotImplementedException(), - }; - } - private static Mock GetProperties( - string checkpointerPath, string transferId, string sourcePath, string destinationPath, - string sourceResourceId, - string destinationResourceId, bool isContainer) { UriBuilder sourceBuilder = new UriBuilder() @@ -59,86 +45,14 @@ private static Mock GetProperties( var mock = new Mock(MockBehavior.Strict); mock.Setup(p => p.TransferId).Returns(transferId); - mock.Setup(p => p.Checkpointer).Returns(new TransferCheckpointStoreOptions(checkpointerPath)); mock.Setup(p => p.SourceUri).Returns(sourceBuilder.Uri); mock.Setup(p => p.DestinationUri).Returns(destinationBuilder.Uri); - mock.Setup(p => p.SourceTypeId).Returns(sourceResourceId); - mock.Setup(p => p.DestinationTypeId).Returns(destinationResourceId); + mock.Setup(p => p.SourceProviderId).Returns("local"); + mock.Setup(p => p.DestinationProviderId).Returns("local"); mock.Setup(p => p.IsContainer).Returns(isContainer); return mock; } - private async Task AddJobPartToCheckpointer( - TransferCheckpointer checkpointer, - string transferId, - StorageResourceType sourceType, - List sourcePaths, - StorageResourceType destinatonType, - List destinationPaths, - int partCount = 1) - { - // Populate sourcePaths if not provided - if (sourcePaths == default) - { - string sourcePath = "sample-source"; - sourcePaths = new List(); - for (int i = 0; i < partCount; i++) - { - sourcePaths.Add(Path.Combine(sourcePath, $"file{i}")); - } - } - // Populate destPaths if not provided - if (destinationPaths == default) - { - string destPath = "sample-dest"; - destinationPaths = new List(); - for (int i = 0; i < partCount; i++) - { - destinationPaths.Add(Path.Combine(destPath, $"file{i}")); - } - } - - JobPlanOperation operationType; - if (sourceType == StorageResourceType.Local) - { - operationType = JobPlanOperation.Upload; - } - else if (destinatonType == StorageResourceType.Local) - { - operationType = JobPlanOperation.Download; - } - else - { - operationType = JobPlanOperation.ServiceToService; - } - - // Use dummy resources that don't correspond to correct paths - StorageResource source = MockStorageResource.MakeSourceResource(10); - StorageResource destination = MockStorageResource.MakeDestinationResource(); - await checkpointer.AddNewJobAsync(transferId, source, destination); - - for (int currentPart = 0; currentPart < partCount; currentPart++) - { - JobPartPlanHeader header = CheckpointerTesting.CreateDefaultJobPartHeader( - transferId: transferId, - partNumber: currentPart, - sourcePath: sourcePaths[currentPart], - destinationPath: destinationPaths[currentPart], - fromTo: operationType); - - using (Stream stream = new MemoryStream()) - { - header.Serialize(stream); - - await checkpointer.AddNewJobPartAsync( - transferId: transferId, - partNumber: currentPart, - chunksTotal: 1, - headerStream: stream); - } - } - } - [Test] public async Task RehydrateLocalFile( [Values(true, false)] bool isSource) @@ -155,22 +69,11 @@ public async Task RehydrateLocalFile( StorageResourceType destinationType = isSource ? StorageResourceType.BlockBlob : StorageResourceType.Local; DataTransferProperties transferProperties = GetProperties( - test.DirectoryPath, transferId, sourcePath, destinationPath, - ToResourceId(sourceType), - ToResourceId(destinationType), isContainer: false).Object; - await AddJobPartToCheckpointer( - checkpointer, - transferId, - sourceType, - new List() { sourcePath }, - destinationType, - new List() { destinationPath } ); - StorageResource storageResource = isSource ? await new LocalFilesStorageResourceProvider().FromSourceAsync(transferProperties, CancellationToken.None) : await new LocalFilesStorageResourceProvider().FromDestinationAsync(transferProperties, CancellationToken.None); @@ -204,23 +107,11 @@ public async Task RehydrateLocalDirectory( StorageResourceType destinationType = isSource ? StorageResourceType.BlockBlob : StorageResourceType.Local; DataTransferProperties transferProperties = GetProperties( - test.DirectoryPath, transferId, sourceParentPath, destinationParentPath, - ToResourceId(sourceType), - ToResourceId(destinationType), isContainer: true).Object; - await AddJobPartToCheckpointer( - checkpointer, - transferId, - sourceType, - sourcePaths, - destinationType, - destinationPaths, - jobPartCount); - StorageResource storageResource = isSource ? await new LocalFilesStorageResourceProvider().FromSourceAsync(transferProperties, CancellationToken.None) : await new LocalFilesStorageResourceProvider().FromDestinationAsync(transferProperties, CancellationToken.None); diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryStorageResourceContainer.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryStorageResourceContainer.cs index 978e75fa39d7..757a7daea9d2 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryStorageResourceContainer.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryStorageResourceContainer.cs @@ -54,12 +54,12 @@ protected internal override async IAsyncEnumerable GetStorageRe } } - public override StorageResourceCheckpointData GetDestinationCheckpointData() + protected internal override StorageResourceCheckpointData GetDestinationCheckpointData() { throw new NotImplementedException(); } - public override StorageResourceCheckpointData GetSourceCheckpointData() + protected internal override StorageResourceCheckpointData GetSourceCheckpointData() { throw new NotImplementedException(); } diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryStorageResourceItem.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryStorageResourceItem.cs index 410dfc531a17..7a97cebe4be7 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryStorageResourceItem.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryStorageResourceItem.cs @@ -68,17 +68,17 @@ protected internal override Task GetCopyAuthorizationHeaderAs throw new NotImplementedException(); } - public override StorageResourceCheckpointData GetDestinationCheckpointData() + protected internal override Task GetPropertiesAsync(CancellationToken token = default) { - throw new NotImplementedException(); + return Task.FromResult(new StorageResourceProperties(default, default, Buffer.Length, default)); } - protected internal override Task GetPropertiesAsync(CancellationToken token = default) + protected internal override StorageResourceCheckpointData GetDestinationCheckpointData() { - return Task.FromResult(new StorageResourceProperties(default, default, Buffer.Length, default)); + throw new NotImplementedException(); } - public override StorageResourceCheckpointData GetSourceCheckpointData() + protected internal override StorageResourceCheckpointData GetSourceCheckpointData() { throw new NotImplementedException(); }