diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/assets.json b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/assets.json index ae6413a7c772..d696fe3b8596 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/assets.json +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.DataMovement.Files.Shares", - "Tag": "net/storage/Azure.Storage.DataMovement.Files.Shares_29f88351cb" + "Tag": "net/storage/Azure.Storage.DataMovement.Files.Shares_02f417687f" } diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs index efff697fc3c8..1ed606f4374b 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs @@ -158,12 +158,16 @@ protected override async Task CopyFromUriAsync( CancellationToken cancellationToken = default) { CancellationHelper.ThrowIfCancellationRequested(cancellationToken); - await ShareFileClient.UploadRangeFromUriAsync( - sourceUri: sourceResource.Uri, - range: new HttpRange(0, completeLength), - sourceRange: new HttpRange(0, completeLength), - options: _options.ToShareFileUploadRangeFromUriOptions(), - cancellationToken: cancellationToken).ConfigureAwait(false); + await CreateAsync(overwrite, completeLength, cancellationToken).ConfigureAwait(false); + if (completeLength > 0) + { + await ShareFileClient.UploadRangeFromUriAsync( + sourceUri: sourceResource.Uri, + range: new HttpRange(0, completeLength), + sourceRange: new HttpRange(0, completeLength), + options: _options.ToShareFileUploadRangeFromUriOptions(), + cancellationToken: cancellationToken).ConfigureAwait(false); + } } protected override async Task DeleteIfExistsAsync(CancellationToken cancellationToken = default) diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj index 301c83b03a61..de152cde090f 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj @@ -44,6 +44,7 @@ + diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ClientBuilderExtensions.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ClientBuilderExtensions.cs index 23267959a0fa..973d321c3e8e 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ClientBuilderExtensions.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ClientBuilderExtensions.cs @@ -10,6 +10,7 @@ using SharesClientBuilder = Azure.Storage.Test.Shared.ClientBuilder< Azure.Storage.Files.Shares.ShareServiceClient, Azure.Storage.Files.Shares.ShareClientOptions>; +using Azure.Storage.Files.Shares.Models; namespace Azure.Storage.DataMovement.Files.Shares.Tests { @@ -42,6 +43,17 @@ public static SharesClientBuilder GetNewShareClientBuilder(TenantConfigurationBu (uri, azureSasCredential, clientOptions) => new ShareServiceClient(uri, azureSasCredential, clientOptions), () => new ShareClientOptions(serviceVersion)); + public static ShareServiceClient GetServiceClient_OAuthAccount_SharedKey(this SharesClientBuilder clientBuilder) => + clientBuilder.GetServiceClientFromSharedKeyConfig(clientBuilder.Tenants.TestConfigOAuth); + + public static ShareServiceClient GetServiceClient_OAuth( + this SharesClientBuilder clientBuilder, ShareClientOptions options = default) + { + options ??= clientBuilder.GetOptions(); + options.ShareTokenIntent = ShareTokenIntent.Backup; + return clientBuilder.GetServiceClientFromOauthConfig(clientBuilder.Tenants.TestConfigOAuth, options); + } + public static async Task GetTestShareAsync( this SharesClientBuilder clientBuilder, ShareServiceClient service = default, diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareFileResourceTests.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareFileResourceTests.cs index 163948699234..6ba31a1b39ab 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareFileResourceTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareFileResourceTests.cs @@ -313,6 +313,22 @@ public async Task CopyFromUriAsync() contentHash: default, isServerEncrypted: false), new MockResponse(200)))); + mockDestination.Setup(b => b.ExistsAsync(It.IsAny())) + .Returns(Task.FromResult(Response.FromValue(false, new MockResponse(200)))); + mockDestination.Setup(b => b.CreateAsync(It.IsAny(), It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + FilesModelFactory.StorageFileInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + isServerEncrypted: false, + filePermissionKey: "rw", + fileAttributes: "Archive|ReadOnly", + fileCreationTime: DateTimeOffset.UtcNow, + fileLastWriteTime: DateTimeOffset.UtcNow, + fileChangeTime: DateTimeOffset.UtcNow, + fileId: "48903841", + fileParentId: "93024923"), + new MockResponse(200)))); ShareFileStorageResource destinationResource = new ShareFileStorageResource(mockDestination.Object); int length = 1024; @@ -327,6 +343,18 @@ public async Task CopyFromUriAsync() It.IsAny(), It.IsAny()), Times.Once()); + mockDestination.Verify(b => b.CreateAsync( + length, + It.IsAny(), + It.IsAny>(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Once()); + mockDestination.Verify(b => b.ExistsAsync( + It.IsAny()), + Times.Once()); mockDestination.VerifyNoOtherCalls(); } @@ -342,28 +370,24 @@ public async Task CopyFromUriAsync_Error() new Uri("https://storageaccount.file.core.windows.net/container/destinationfile"), new ShareClientOptions()); - mockDestination.Setup(b => b.UploadRangeFromUriAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) - .Throws(new RequestFailedException(status: 404, message: "The specified resource does not exist.", errorCode: "ResourceNotFound", default)); + mockDestination.Setup(b => b.ExistsAsync(It.IsAny())) + .Returns(Task.FromResult(Response.FromValue(true, new MockResponse(200)))); ShareFileStorageResource destinationResource = new ShareFileStorageResource(mockDestination.Object); // Act int length = 1024; - await TestHelper.AssertExpectedExceptionAsync( - destinationResource.CopyFromUriInternalAsync(sourceResource.Object, false, length), + await TestHelper.AssertExpectedExceptionAsync( + destinationResource.CopyBlockFromUriInternalAsync(sourceResource.Object, new HttpRange(0, length), false, length), e => { - Assert.AreEqual("ResourceNotFound", e.ErrorCode); + Assert.IsTrue(e.Message.Contains("Cannot overwrite file.")); }); - sourceResource.Verify(b => b.Uri, Times.Once()); sourceResource.VerifyNoOtherCalls(); - mockDestination.Verify(b => b.UploadRangeFromUriAsync( - sourceResource.Object.Uri, - new HttpRange(0, length), - new HttpRange(0, length), - It.IsAny(), + mockDestination.Verify(b => b.ExistsAsync( It.IsAny()), Times.Once()); + mockDestination.Verify(b => b.Path, Times.Once()); mockDestination.VerifyNoOtherCalls(); } diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareFileStartTransferCopyTests.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareFileStartTransferCopyTests.cs new file mode 100644 index 000000000000..d939844f64ec --- /dev/null +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareFileStartTransferCopyTests.cs @@ -0,0 +1,142 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Threading.Tasks; +using Azure.Storage.Test.Shared; +using Azure.Storage.DataMovement.Tests; +using Azure.Storage.Files.Shares; +using Azure.Storage.Files.Shares.Tests; +using System.IO; +using Azure.Core; +using Azure.Core.TestFramework; + +namespace Azure.Storage.DataMovement.Files.Shares.Tests +{ + [ShareClientTestFixture] + public class ShareFileStartTransferCopyTests : StartTransferCopyTestBase + + { + private const string _fileResourcePrefix = "test-file-"; + private const string _expectedOverwriteExceptionMessage = "Cannot overwrite file."; + protected readonly ShareClientOptions.ServiceVersion _serviceVersion; + + public ShareFileStartTransferCopyTests(bool async, ShareClientOptions.ServiceVersion serviceVersion) + : base(async, _expectedOverwriteExceptionMessage, _fileResourcePrefix, null /* RecordedTestMode.Record /* to re-record */) + { + _serviceVersion = serviceVersion; + SourceClientBuilder = ClientBuilderExtensions.GetNewShareClientBuilder(Tenants, serviceVersion); + DestinationClientBuilder = ClientBuilderExtensions.GetNewShareClientBuilder(Tenants, serviceVersion); + } + + protected override async Task SourceExistsAsync(ShareFileClient objectClient) + => await objectClient.ExistsAsync(); + + protected override async Task DestinationExistsAsync(ShareFileClient objectClient) + => await objectClient.ExistsAsync(); + + protected override async Task> GetSourceDisposingContainerAsync(ShareServiceClient service = null, string containerName = null) + => await SourceClientBuilder.GetTestShareAsync(service, containerName); + + protected override async Task> GetDestinationDisposingContainerAsync(ShareServiceClient service = null, string containerName = null) + => await DestinationClientBuilder.GetTestShareAsync(service, containerName); + + private async Task CreateFileClientAsync( + ShareClient container, + long? objectLength = null, + bool createResource = false, + string objectName = null, + ShareClientOptions options = null, + Stream contents = null) + { + objectName ??= GetNewObjectName(); + ShareFileClient fileClient = container.GetRootDirectoryClient().GetFileClient(objectName); + if (createResource) + { + if (!objectLength.HasValue) + { + throw new InvalidOperationException($"Cannot create share file without size specified. Either set {nameof(createResource)} to false or specify a {nameof(objectLength)}."); + } + await fileClient.CreateAsync(objectLength.Value); + + if (contents != default) + { + await fileClient.UploadAsync(contents); + } + } + Uri sourceUri = fileClient.GenerateSasUri(Sas.ShareFileSasPermissions.All, Recording.UtcNow.AddDays(1)); + return InstrumentClient(new ShareFileClient(sourceUri, GetOptions())); + } + + protected override Task GetSourceObjectClientAsync( + ShareClient container, + long? objectLength = null, + bool createResource = false, + string objectName = null, + ShareClientOptions options = null, + Stream contents = null) + => CreateFileClientAsync( + container, + objectLength, + createResource, + objectName, + options, + contents); + + protected override StorageResourceItem GetSourceStorageResourceItem(ShareFileClient objectClient) + => new ShareFileStorageResource(objectClient); + + protected override Task SourceOpenReadAsync(ShareFileClient objectClient) + => objectClient.OpenReadAsync(); + + protected override Task GetDestinationObjectClientAsync( + ShareClient container, + long? objectLength = null, + bool createResource = false, + string objectName = null, + ShareClientOptions options = null, + Stream contents = null) + => CreateFileClientAsync( + container, + objectLength, + createResource, + objectName, + options, + contents); + + protected override StorageResourceItem GetDestinationStorageResourceItem(ShareFileClient objectClient) + => new ShareFileStorageResource(objectClient); + + protected override Task DestinationOpenReadAsync(ShareFileClient objectClient) + => objectClient.OpenReadAsync(); + + public ShareClientOptions GetOptions() + { + var options = new ShareClientOptions(_serviceVersion) + { + Diagnostics = { IsLoggingEnabled = true }, + Retry = + { + Mode = RetryMode.Exponential, + MaxRetries = Constants.MaxReliabilityRetries, + Delay = TimeSpan.FromSeconds(Mode == RecordedTestMode.Playback ? 0.01 : 1), + MaxDelay = TimeSpan.FromSeconds(Mode == RecordedTestMode.Playback ? 0.1 : 60) + }, + }; + if (Mode != RecordedTestMode.Live) + { + options.AddPolicy(new RecordedClientRequestIdPolicy(Recording), HttpPipelinePosition.PerCall); + } + + return InstrumentClientOptions(options); + } + } +} diff --git a/sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceJobPart.cs b/sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceJobPart.cs index 258380048d50..68411abee85f 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceJobPart.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceJobPart.cs @@ -210,6 +210,12 @@ await _destinationResource.CopyFromUriAsync( { await InvokeSkippedArg().ConfigureAwait(false); } + catch (InvalidOperationException ex) + when (_createMode == StorageResourceCreationPreference.SkipIfExists + && ex.Message.Contains("Cannot overwrite file.")) + { + await InvokeSkippedArg().ConfigureAwait(false); + } catch (Exception ex) { await InvokeFailedArg(ex).ConfigureAwait(false); diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferCopyTestBase.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferCopyTestBase.cs new file mode 100644 index 000000000000..9c95838acca5 --- /dev/null +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferCopyTestBase.cs @@ -0,0 +1,859 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Azure.Core.TestFramework; +using Azure.Core; +using Azure.Storage.Test.Shared; +using System.Threading.Tasks; +using System.IO; +using NUnit.Framework; +using System.Threading; + +namespace Azure.Storage.DataMovement.Tests +{ + public abstract class StartTransferCopyTestBase + : StorageTestBase + where TSourceServiceClient : class + where TSourceContainerClient : class + where TSourceObjectClient : class + where TSourceClientOptions : ClientOptions + where TDestinationServiceClient : class + where TDestinationContainerClient : class + where TDestinationObjectClient : class + where TDestinationClientOptions : ClientOptions + where TEnvironment : StorageTestEnvironment, new() + { + private readonly string _generatedResourceNamePrefix; + private readonly string _expectedOverwriteExceptionMessage; + + public ClientBuilder SourceClientBuilder { get; protected set; } + public ClientBuilder DestinationClientBuilder { get; protected set; } + + /// + /// Constructor for TransferManager.StartTransferAsync tests + /// + /// The async is defaulted to true, since we do not have sync StartTransfer methods. + /// + /// + /// + public StartTransferCopyTestBase( + bool async, + string expectedOverwriteExceptionMessage, + string generatedResourceNamePrefix = default, + RecordedTestMode? mode = null) : base(async, mode) + { + Argument.CheckNotNullOrEmpty(expectedOverwriteExceptionMessage, nameof(expectedOverwriteExceptionMessage)); + _generatedResourceNamePrefix = generatedResourceNamePrefix ?? "test-resource-"; + _expectedOverwriteExceptionMessage = expectedOverwriteExceptionMessage; + } + + #region Service-Specific Methods + /// + /// Gets a service-specific disposing container for use with tests in this class. + /// + /// Optionally specified service client to get container from. + /// Optional container name specification. + protected abstract Task> GetSourceDisposingContainerAsync( + TSourceServiceClient service = default, + string containerName = default); + + /// + /// Gets a new service-specific child object client from a given container, e.g. a BlobClient from a + /// TSourceContainerClient or a TSourceObjectClient from a ShareClient. + /// + /// Container to get resource from. + /// Sets the resource size in bytes, for resources that require this upfront. + /// Whether to call CreateAsync on the resource, if necessary. + /// Optional name for the resource. + /// ClientOptions for the resource client. + /// If specified, the contents will be uploaded to the object client. + protected abstract Task GetSourceObjectClientAsync( + TSourceContainerClient container, + long? objectLength = default, + bool createResource = false, + string objectName = default, + TSourceClientOptions options = default, + Stream contents = default); + + /// + /// Gets the specific storage resource from the given TSourceObjectClient + /// e.g. ShareFileClient to a ShareFileStorageResource, TSourceObjectClient to a BlockBlobStorageResource. + /// + /// The object client to create the storage resource object. + /// + protected abstract StorageResourceItem GetSourceStorageResourceItem(TSourceObjectClient objectClient); + + /// + /// Calls the OpenRead method on the TSourceObjectClient. + /// + /// This is mainly used to verify the contents of the Object Client. + /// + /// The object client to get the Open Read Stream from. + /// + protected abstract Task SourceOpenReadAsync(TSourceObjectClient objectClient); + + /// + /// Checks if the Object Client exists. + /// + /// Object Client to call exists on. + /// + protected abstract Task SourceExistsAsync(TSourceObjectClient objectClient); + + /// + /// Gets a service-specific disposing container for use with tests in this class. + /// + /// Optionally specified service client to get container from. + /// Optional container name specification. + protected abstract Task> GetDestinationDisposingContainerAsync( + TDestinationServiceClient service = default, + string containerName = default); + + /// + /// Gets a new service-specific child object client from a given container, e.g. a BlobClient from a + /// TSourceContainerClient or a TDestinationObjectClient from a ShareClient. + /// + /// Container to get resource from. + /// Sets the resource size in bytes, for resources that require this upfront. + /// Whether to call CreateAsync on the resource, if necessary. + /// Optional name for the resource. + /// ClientOptions for the resource client. + /// If specified, the contents will be uploaded to the object client. + protected abstract Task GetDestinationObjectClientAsync( + TDestinationContainerClient container, + long? objectLength = default, + bool createResource = false, + string objectName = default, + TDestinationClientOptions options = default, + Stream contents = default); + + /// + /// Gets the specific storage resource from the given TDestinationObjectClient + /// e.g. ShareFileClient to a ShareFileStorageResource, TSourceObjectClient to a BlockBlobStorageResource. + /// + /// The object client to create the storage resource object. + /// + protected abstract StorageResourceItem GetDestinationStorageResourceItem(TDestinationObjectClient objectClient); + + /// + /// Calls the OpenRead method on the TDestinationObjectClient. + /// + /// This is mainly used to verify the contents of the Object Client. + /// + /// The object client to get the Open Read Stream from. + /// + protected abstract Task DestinationOpenReadAsync(TDestinationObjectClient objectClient); + + /// + /// Checks if the Object Client exists. + /// + /// Object Client to call exists on. + /// + protected abstract Task DestinationExistsAsync(TDestinationObjectClient objectClient); + #endregion + + protected string GetNewObjectName() + => _generatedResourceNamePrefix + SourceClientBuilder.Recording.Random.NewGuid(); + + internal class VerifyObjectCopyFromUriInfo + { + public readonly string SourceLocalPath; + public readonly StorageResourceItem SourceResource; + public readonly TSourceObjectClient SourceClient; + public readonly StorageResourceItem DestinationResource; + public readonly TDestinationObjectClient DestinationClient; + public TestEventsRaised testEventsRaised; + public DataTransfer DataTransfer; + public bool CompletedStatus; + + public VerifyObjectCopyFromUriInfo( + string sourceLocalPath, + StorageResourceItem sourceResource, + TSourceObjectClient sourceClient, + StorageResourceItem destinationResource, + TDestinationObjectClient destinationClient, + TestEventsRaised eventsRaised, + bool completed) + { + SourceLocalPath = sourceLocalPath; + SourceResource = sourceResource; + SourceClient = sourceClient; + DestinationResource = destinationResource; + DestinationClient = destinationClient; + testEventsRaised = eventsRaised; + CompletedStatus = completed; + DataTransfer = default; + } + }; + #region Copy RemoteObject + /// + /// Upload the source remote object, then copy the contents to another remote object. + /// Then copy the remote object and verify the contents. + /// + /// By default in this function an event argument will be added to the options event handler + /// to detect when the upload has finished. + /// + /// + /// + /// + /// + private async Task CopyRemoteObjectsAndVerify( + TSourceContainerClient sourceContainer, + TDestinationContainerClient destinationContainer, + long size = Constants.KB, + int waitTimeInSec = 30, + int objectCount = 1, + TransferManagerOptions transferManagerOptions = default, + List sourceObjectNames = default, + List destinationObjectNames = default, + List options = default) + { + using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); + // Populate objectCount list for number of objects to be created + if (sourceObjectNames == default || sourceObjectNames?.Count == 0) + { + sourceObjectNames ??= new List(); + for (int i = 0; i < objectCount; i++) + { + sourceObjectNames.Add(GetNewObjectName()); + } + } + else + { + // If objectNames is populated make sure these number of objects match + Assert.AreEqual(objectCount, sourceObjectNames.Count); + } + + // Populate objectNames list for number of objects to be created + if (destinationObjectNames == default || destinationObjectNames?.Count == 0) + { + destinationObjectNames ??= new List(); + for (int i = 0; i < objectCount; i++) + { + destinationObjectNames.Add(GetNewObjectName()); + } + } + else + { + // If objectNames is populated make sure these number of objects match + Assert.AreEqual(objectCount, destinationObjectNames.Count); + } + + // Populate Options and TestRaisedOptions + List eventsRaisedList = TestEventsRaised.PopulateTestOptions(objectCount, ref options); + + transferManagerOptions ??= new TransferManagerOptions() + { + ErrorHandling = DataTransferErrorMode.ContinueOnFailure + }; + + List copyObjectInfo = new List(objectCount); + // Initialize transfer manager + TransferManager transferManager = new TransferManager(transferManagerOptions); + + // Upload set of VerifyCopyFromUriInfo Remote Objects to Copy + for (int i = 0; i < objectCount; i++) + { + bool completed = false; + // Set up object to be Copied + var data = GetRandomBuffer(size); + using Stream originalStream = await CreateLimitedMemoryStream(size); + string localSourceFile = Path.Combine(testDirectory.DirectoryPath, sourceObjectNames[i]); + TSourceObjectClient sourceClient = await GetSourceObjectClientAsync( + container: sourceContainer, + objectLength: size, + createResource: true, + objectName: sourceObjectNames[i]); + + StorageResourceItem sourceResource = GetSourceStorageResourceItem(sourceClient); + // Set up destination client + TDestinationObjectClient destClient = await GetDestinationObjectClientAsync( + container: destinationContainer, + createResource: false, + objectName: string.Concat(destinationObjectNames[i])); + StorageResourceItem destinationResource = GetDestinationStorageResourceItem(destClient); + copyObjectInfo.Add(new VerifyObjectCopyFromUriInfo( + localSourceFile, + sourceResource, + sourceClient, + destinationResource, + destClient, + eventsRaisedList[i], + completed)); + } + + // Schedule all Copy Remote Objects consecutively + for (int i = 0; i < copyObjectInfo.Count; i++) + { + // Act + DataTransfer transfer = await transferManager.StartTransferAsync( + copyObjectInfo[i].SourceResource, + copyObjectInfo[i].DestinationResource, + options[i]).ConfigureAwait(false); + copyObjectInfo[i].DataTransfer = transfer; + } + + for (int i = 0; i < copyObjectInfo.Count; i++) + { + // Assert + Assert.NotNull(copyObjectInfo[i].DataTransfer); + CancellationTokenSource tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(waitTimeInSec)); + await copyObjectInfo[i].DataTransfer.WaitForCompletionAsync(tokenSource.Token); + Assert.IsTrue(copyObjectInfo[i].DataTransfer.HasCompleted); + Assert.AreEqual(DataTransferState.Completed, copyObjectInfo[i].DataTransfer.TransferStatus.State); + + // Verify Copy - using original source File and Copying the destination + await copyObjectInfo[i].testEventsRaised.AssertSingleCompletedCheck(); + using Stream sourceStream = await SourceOpenReadAsync(copyObjectInfo[i].SourceClient); + using Stream destinationStream = await DestinationOpenReadAsync(copyObjectInfo[i].DestinationClient); + Assert.AreEqual(sourceStream, destinationStream); + } + } + + [RecordedTest] + public async Task SourceObjectToDestinationObject() + { + // Arrange + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); + + // No Option Copy bag or manager options bag, plain Copy + await CopyRemoteObjectsAndVerify(source.Container, destination.Container).ConfigureAwait(false); + } + + [RecordedTest] + public async Task SourceObjectToDestinationObject_SmallChunk() + { + long size = Constants.KB; + int waitTimeInSec = 25; + + DataTransferOptions options = new DataTransferOptions() + { + InitialTransferSize = 100, + MaximumTransferChunkSize = 200, + }; + + // Arrange + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); + + List optionsList = new List() { options }; + await CopyRemoteObjectsAndVerify( + source.Container, + destination.Container, + waitTimeInSec: waitTimeInSec, + size: size, + options: optionsList).ConfigureAwait(false); + } + + [RecordedTest] + [TestCase(0, 10)] + [TestCase(100, 10)] + [TestCase(Constants.KB, 10)] + [TestCase(2 * Constants.KB, 10)] + public async Task SourceObjectToDestinationObject_SmallSize(long size, int waitTimeInSec) + { + // Arrange + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); + + await CopyRemoteObjectsAndVerify( + source.Container, + destination.Container, + size: size, + waitTimeInSec: waitTimeInSec).ConfigureAwait(false); + } + + [Ignore("These tests currently take 40+ mins for little additional coverage")] + [Test] + [LiveOnly] + [TestCase(4 * Constants.MB, 20)] + [TestCase(5 * Constants.MB, 20)] + [TestCase(257 * Constants.MB, 400)] + [TestCase(Constants.GB, 1000)] + public async Task SourceObjectToDestinationObject_LargeSize(long size, int waitTimeInSec) + { + // Arrange + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); + + await CopyRemoteObjectsAndVerify( + source.Container, + destination.Container, + size: size, + waitTimeInSec: waitTimeInSec).ConfigureAwait(false); + } + + [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/33003")] + [Test] + [LiveOnly] + [TestCase(2, 0, 30)] + [TestCase(6, 0, 30)] + [TestCase(2, 100, 30)] + [TestCase(6, 100, 30)] + [TestCase(2, Constants.KB, 300)] + [TestCase(6, Constants.KB, 300)] + public async Task SourceObjectToDestinationObject_SmallMultiple(int count, long size, int waitTimeInSec) + { + // Arrange + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); + + await CopyRemoteObjectsAndVerify( + source.Container, + destination.Container, + objectCount: count, + size: size, + waitTimeInSec: waitTimeInSec).ConfigureAwait(false); + } + + [Ignore("These tests currently take 40+ mins for little additional coverage")] + [Test] + [LiveOnly] + [TestCase(2, 4 * Constants.MB, 300)] + [TestCase(6, 4 * Constants.MB, 300)] + [TestCase(2, 257 * Constants.MB, 400)] + [TestCase(6, 257 * Constants.MB, 600)] + [TestCase(2, Constants.GB, 2000)] + public async Task SourceObjectToDestinationObject_LargeMultiple(int count, long size, int waitTimeInSec) + { + // Arrange + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); + + await CopyRemoteObjectsAndVerify( + source.Container, + destination.Container, + objectCount: count, + size: size, + waitTimeInSec: waitTimeInSec).ConfigureAwait(false); + } + + [RecordedTest] + public async Task SourceObjectToDestinationObject_Overwrite_Exists() + { + // Arrange + // Create source local file for checking, and source object + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); + + using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); + string name = GetNewObjectName(); + string localSourceFile = Path.Combine(testDirectory.DirectoryPath, name); + int size = Constants.KB; + // Create destination, so when we attempt to transfer, we have something to overwrite. + TDestinationObjectClient destClient = await GetDestinationObjectClientAsync( + container: destination.Container, + objectName: name, + objectLength: size, + createResource: true); + + // Act + // Create options bag to overwrite any existing destination. + DataTransferOptions options = new DataTransferOptions() + { + CreationPreference = StorageResourceCreationPreference.OverwriteIfExists, + }; + List optionsList = new List() { options }; + List objectNames = new List() { name }; + + // Start transfer and await for completion. + await CopyRemoteObjectsAndVerify( + source.Container, + destination.Container, + destinationObjectNames: objectNames, + options: optionsList); + } + + [RecordedTest] + public async Task SourceObjectToDestinationObject_Overwrite_NotExists() + { + // Arrange + // Create source local file for checking, and source object + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); + + int size = Constants.KB; + int waitTimeInSec = 10; + + // Act + // Create options bag to overwrite any existing destination. + DataTransferOptions options = new DataTransferOptions() + { + CreationPreference = StorageResourceCreationPreference.OverwriteIfExists, + }; + List optionsList = new List() { options }; + + // Start transfer and await for completion. + await CopyRemoteObjectsAndVerify( + source.Container, + destination.Container, + size: size, + waitTimeInSec: waitTimeInSec, + options: optionsList); + } + + [RecordedTest] + public async Task SourceObjectToDestinationObject_Skip_Exists() + { + // Arrange + // Create source local file for checking, and source object + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); + + using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); + string objectName = GetNewObjectName(); + string originalSourceFile = Path.Combine(testDirectory.DirectoryPath, objectName); + int size = Constants.KB; + TDestinationObjectClient destinationClient = await GetDestinationObjectClientAsync( + destination.Container, + objectName: objectName, + objectLength: size, + createResource: true); + + // Act + // Create options bag to overwrite any existing destination. + DataTransferOptions options = new DataTransferOptions() + { + CreationPreference = StorageResourceCreationPreference.SkipIfExists, + }; + + // Create new source block object. + string newSourceFile = Path.Combine(testDirectory.DirectoryPath, GetNewObjectName()); + TSourceObjectClient sourceClient = await GetSourceObjectClientAsync( + source.Container, + objectName: GetNewObjectName(), + objectLength: size, + createResource: true); + StorageResourceItem sourceResource = GetSourceStorageResourceItem(sourceClient); + StorageResourceItem destinationResource = GetDestinationStorageResourceItem(destinationClient); + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + + TransferManager transferManager = new TransferManager(); + + // Start transfer and await for completion. + DataTransfer transfer = await transferManager.StartTransferAsync( + sourceResource, + destinationResource, + options); + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await transfer.WaitForCompletionAsync(cancellationTokenSource.Token); + + // Assert + Assert.NotNull(transfer); + Assert.IsTrue(transfer.HasCompleted); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasSkippedItems); + await testEventsRaised.AssertSingleSkippedCheck(); + Assert.IsTrue(await DestinationExistsAsync(destinationClient)); + // Verify Upload - That we skipped over and didn't reupload something new. + using Stream sourceStream = await SourceOpenReadAsync(sourceClient); + using Stream destinationStream = await DestinationOpenReadAsync(destinationClient); + Assert.AreEqual(sourceStream, destinationStream); + } + + [RecordedTest] + public async Task SourceObjectToDestinationObject_Failure_Exists() + { + // Arrange + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); + using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); + string name = GetNewObjectName(); + string originalSourceFile = Path.Combine(testDirectory.DirectoryPath, name); + int size = Constants.KB; + TDestinationObjectClient destinationClient = await GetDestinationObjectClientAsync( + container: destination.Container, + objectLength: size, + createResource: true, + objectName: name); + + // Act + // Create options bag to fail and keep track of the failure. + DataTransferOptions options = new DataTransferOptions() + { + CreationPreference = StorageResourceCreationPreference.FailIfExists, + }; + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + // Create new source object. + string newSourceFile = Path.Combine(testDirectory.DirectoryPath, GetNewObjectName()); + TSourceObjectClient sourceClient = await GetSourceObjectClientAsync( + container: source.Container, + objectName: GetNewObjectName(), + createResource: true, + objectLength: size); + StorageResourceItem sourceResource = GetSourceStorageResourceItem(sourceClient); + StorageResourceItem destinationResource = GetDestinationStorageResourceItem(destinationClient); + TransferManager transferManager = new TransferManager(); + + // Start transfer and await for completion. + DataTransfer transfer = await transferManager.StartTransferAsync( + sourceResource, + destinationResource, + options); + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await transfer.WaitForCompletionAsync(cancellationTokenSource.Token); + + // Assert + Assert.NotNull(transfer); + Assert.IsTrue(transfer.HasCompleted); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); + Assert.IsTrue(await DestinationExistsAsync(destinationClient)); + await testEventsRaised.AssertSingleFailedCheck(); + Assert.NotNull(testEventsRaised.FailedEvents.First().Exception, "Excepted failure: Overwrite failure was supposed to be raised during the test"); + Assert.IsTrue(testEventsRaised.FailedEvents.First().Exception.Message.Contains(_expectedOverwriteExceptionMessage)); + // Verify Upload - That we skipped over and didn't reupload something new. + using Stream sourceStream = await SourceOpenReadAsync(sourceClient); + using Stream destinationStream = await DestinationOpenReadAsync(destinationClient); + Assert.AreEqual(sourceStream, destinationStream); + } + #endregion + + private async Task CreateStartTransfer( + TSourceContainerClient sourceContainer, + TDestinationContainerClient destinationContainer, + int concurrency, + bool createFailedCondition = false, + DataTransferOptions options = default, + int size = Constants.KB) + { + // Arrange + // Create source local file for checking, and source object + string sourceName = GetNewObjectName(); + string destinationName = GetNewObjectName(); + using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); + TDestinationObjectClient destinationClient = await GetDestinationObjectClientAsync( + container: destinationContainer, + createResource: createFailedCondition, + objectName: destinationName, + objectLength: size); + + // Create new source object. + string newSourceFile = Path.Combine(testDirectory.DirectoryPath, sourceName); + TSourceObjectClient sourceClient = await GetSourceObjectClientAsync( + container: sourceContainer, + objectName: sourceName, + createResource: true, + objectLength: size); + StorageResourceItem sourceResource = GetSourceStorageResourceItem(sourceClient); + StorageResourceItem destinationResource = GetDestinationStorageResourceItem(destinationClient); + + // Create Transfer Manager with single threaded operation + TransferManagerOptions managerOptions = new TransferManagerOptions() + { + MaximumConcurrency = concurrency, + }; + TransferManager transferManager = new TransferManager(managerOptions); + + // Start transfer and await for completion. + return await transferManager.StartTransferAsync( + sourceResource, + destinationResource, + options).ConfigureAwait(false); + } + + [RecordedTest] + public async Task StartTransfer_AwaitCompletion() + { + // Arrange + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); + + DataTransferOptions options = new DataTransferOptions(); + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + + // Create transfer to do a AwaitCompletion + DataTransfer transfer = await CreateStartTransfer( + source.Container, + destination.Container, + concurrency: 1, + options: options); + + // Act + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await transfer.WaitForCompletionAsync(cancellationTokenSource.Token).ConfigureAwait(false); + + // Assert + await testEventsRaised.AssertSingleCompletedCheck(); + Assert.NotNull(transfer); + Assert.IsTrue(transfer.HasCompleted); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + } + + [RecordedTest] + public async Task StartTransfer_AwaitCompletion_Failed() + { + // Arrange + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); + + DataTransferOptions options = new DataTransferOptions() + { + CreationPreference = StorageResourceCreationPreference.FailIfExists + }; + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + + // Create transfer to do a AwaitCompletion + DataTransfer transfer = await CreateStartTransfer( + source.Container, + destination.Container, + concurrency: 1, + createFailedCondition: true, + options: options); + + // Act + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await transfer.WaitForCompletionAsync(cancellationTokenSource.Token).ConfigureAwait(false); + + // Assert + Assert.NotNull(transfer); + Assert.IsTrue(transfer.HasCompleted); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); + await testEventsRaised.AssertSingleFailedCheck(); + Assert.IsTrue(testEventsRaised.FailedEvents.First().Exception.Message.Contains(_expectedOverwriteExceptionMessage)); + } + + [RecordedTest] + public async Task StartTransfer_AwaitCompletion_Skipped() + { + // Arrange + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); + + // Create transfer options with Skipping available + DataTransferOptions options = new DataTransferOptions() + { + CreationPreference = StorageResourceCreationPreference.SkipIfExists + }; + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + + // Create transfer to do a AwaitCompletion + DataTransfer transfer = await CreateStartTransfer( + source.Container, + destination.Container, + concurrency: 1, + createFailedCondition: true, + options: options); + + // Act + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await transfer.WaitForCompletionAsync(cancellationTokenSource.Token).ConfigureAwait(false); + + // Assert + await testEventsRaised.AssertSingleSkippedCheck(); + Assert.NotNull(transfer); + Assert.IsTrue(transfer.HasCompleted); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasSkippedItems); + } + + [RecordedTest] + public async Task StartTransfer_EnsureCompleted() + { + // Arrange + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); + + DataTransferOptions options = new DataTransferOptions(); + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + + // Create transfer to do a EnsureCompleted + DataTransfer transfer = await CreateStartTransfer( + source.Container, + destination.Container, + concurrency: 1, + options: options); + + // Act + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + transfer.WaitForCompletion(cancellationTokenSource.Token); + + // Assert + await testEventsRaised.AssertSingleCompletedCheck(); + Assert.NotNull(transfer); + Assert.IsTrue(transfer.HasCompleted); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + } + + [RecordedTest] + public async Task StartTransfer_EnsureCompleted_Failed() + { + // Arrange + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); + + DataTransferOptions options = new DataTransferOptions() + { + CreationPreference = StorageResourceCreationPreference.FailIfExists + }; + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + + // Create transfer to do a AwaitCompletion + DataTransfer transfer = await CreateStartTransfer( + source.Container, + destination.Container, + concurrency: 1, + createFailedCondition: true, + options: options); + + // Act + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + transfer.WaitForCompletion(cancellationTokenSource.Token); + + // Assert + await testEventsRaised.AssertSingleFailedCheck(); + Assert.NotNull(transfer); + Assert.IsTrue(transfer.HasCompleted); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); + Assert.IsTrue(testEventsRaised.FailedEvents.First().Exception.Message.Contains(_expectedOverwriteExceptionMessage)); + } + + [RecordedTest] + public async Task StartTransfer_EnsureCompleted_Skipped() + { + // Arrange + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); + + // Create transfer options with Skipping available + DataTransferOptions options = new DataTransferOptions() + { + CreationPreference = StorageResourceCreationPreference.SkipIfExists + }; + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + + // Create transfer to do a EnsureCompleted + DataTransfer transfer = await CreateStartTransfer( + source.Container, + destination.Container, + concurrency: 1, + createFailedCondition: true, + options: options); + + // Act + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + transfer.WaitForCompletion(cancellationTokenSource.Token); + + // Assert + await testEventsRaised.AssertSingleSkippedCheck(); + Assert.NotNull(transfer); + Assert.IsTrue(transfer.HasCompleted); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasSkippedItems); + } + } +}