diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/TestHelper.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/TestHelper.cs index 93a4b7d7f5f6..1fd08fe518e4 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Shared/TestHelper.cs +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/TestHelper.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading; using System.Threading.Tasks; using Azure.Core.TestFramework; using NUnit.Framework; @@ -162,5 +163,12 @@ public static void AssertInconclusiveRecordingFriendly(RecordedTestMode mode, st Assert.Inconclusive(message); } } + + public static CancellationToken GetTimeoutToken(int seconds) + { + CancellationTokenSource cts = new(); + cts.CancelAfter(TimeSpan.FromSeconds(seconds)); + return cts.Token; + } } } diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferUploadDirectoryTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferUploadDirectoryTests.cs index 5862d00dcbbf..0f490c668a41 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferUploadDirectoryTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferUploadDirectoryTests.cs @@ -13,6 +13,7 @@ using Azure.Storage.Blobs.Models; using Azure.Storage.Blobs.Specialized; using Azure.Storage.Blobs.Tests; +using Azure.Storage.Test; using DMBlobs::Azure.Storage.DataMovement.Blobs; using Microsoft.CodeAnalysis; using NUnit.Framework; @@ -26,24 +27,37 @@ public StartTransferUploadDirectoryTests(bool async, BlobClientOptions.ServiceVe { } #region Directory Block Blob + private async Task SetupDirectory( + string directoryPath, + List<(string FilePath, long Size)> fileSizes, + CancellationToken cancellationToken) + { + foreach ((string filePath, long size) in fileSizes) + { + if (cancellationToken.IsCancellationRequested) + { + return; + } + using FileStream fs = File.OpenWrite(Path.Combine(directoryPath, filePath)); + using Stream data = await CreateLimitedMemoryStream(size); + await data.CopyToAsync(fs, bufferSize: 4 * Constants.KB, cancellationToken); + } + } + /// /// Upload and verify the contents of the blob /// /// By default in this function an event arguement will be added to the options event handler /// to detect when the upload has finished. /// - /// - /// - /// - /// private async Task UploadBlobDirectoryAndVerify( + string sourceLocalDirectoryPath, BlobContainerClient destinationContainer, - string localDirectoryPath, - List files, + int expectedTransfers, string destinationPrefix = default, - int waitTimeInSec = 30, TransferManagerOptions transferManagerOptions = default, - DataTransferOptions options = default) + DataTransferOptions options = default, + CancellationToken cancellationToken = default) { // Set transfer options options ??= new DataTransferOptions(); @@ -54,45 +68,23 @@ private async Task UploadBlobDirectoryAndVerify( ErrorHandling = DataTransferErrorMode.ContinueOnFailure }; - destinationPrefix ??= "foo"; - - // Initialize transferManager - TransferManager transferManager = new TransferManager(transferManagerOptions); - - StorageResourceContainer sourceResource = - new LocalDirectoryStorageResourceContainer(localDirectoryPath); - StorageResourceContainer destinationResource = - new BlobStorageResourceContainer(destinationContainer, new() { BlobDirectoryPrefix = destinationPrefix }); - - // Set up blob to upload - DataTransfer transfer = await transferManager.StartTransferAsync(sourceResource, destinationResource, options); - - // Assert - CancellationTokenSource tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(waitTimeInSec)); - await transfer.WaitForCompletionAsync(tokenSource.Token); - - await testEventsRaised.AssertContainerCompletedCheck(files.Count); - Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); - - // Assert - Check Response - List blobs = ((List)await destinationContainer.GetBlobsAsync(prefix: destinationPrefix).ToListAsync()) - .Select((BlobItem blob) => blob.Name).ToList(); - - // Assert - Check destination blobs - Assert.AreEqual(files.Count, blobs.Count()); + LocalDirectoryStorageResourceContainer sourceResource = new(sourceLocalDirectoryPath); + BlobStorageResourceContainer destinationResource = new(destinationContainer, new() + { + BlobDirectoryPrefix = destinationPrefix + }); - for (int i = 0; i < files.Count; i++) + await new TransferValidator() { - // Verify Upload - using (FileStream fileStream = File.OpenRead(files[i])) - { - string blobName = $"{destinationPrefix}/{files[i].Substring(localDirectoryPath.Length+1)}"; - BlockBlobClient destinationBlob = destinationContainer.GetBlockBlobClient(blobName); - Assert.IsTrue(await destinationBlob.ExistsAsync()); - await DownloadAndAssertAsync(fileStream, destinationBlob); - } - } + TransferManager = new(transferManagerOptions) + }.TransferAndVerifyAsync( + sourceResource, + destinationResource, + TransferValidator.GetLocalFileLister(sourceLocalDirectoryPath), + TransferValidator.GetBlobLister(destinationContainer, destinationPrefix), + expectedTransfers, + options, + cancellationToken); } [Test] @@ -103,26 +95,29 @@ private async Task UploadBlobDirectoryAndVerify( public async Task LocalToBlockBlobDirectory_SmallSize(long blobSize, int waitTimeInSec) { DataTransferOptions options = new DataTransferOptions(); - List files = new List(); using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); string localDirectory = CreateRandomDirectory(testDirectory.DirectoryPath); await using DisposingContainer test = await GetTestContainerAsync(); - files.Add(await CreateRandomFileAsync(localDirectory, size: blobSize)); - files.Add(await CreateRandomFileAsync(localDirectory, size: blobSize)); - - string openSubfolder = CreateRandomDirectory(localDirectory); - files.Add(await CreateRandomFileAsync(openSubfolder, size: blobSize)); - string lockedSubfolder = CreateRandomDirectory(localDirectory); - files.Add(await CreateRandomFileAsync(lockedSubfolder, size: blobSize)); + List files = new() + { + GetNewBlobName(), + GetNewBlobName(), + $"{GetNewBlobName()}/{GetNewBlobName()}", + $"{GetNewBlobName()}/{GetNewBlobName()}", + }; - // Arrange + CancellationToken cancellationToken = TestHelper.GetTimeoutToken(waitTimeInSec); + await SetupDirectory( + localDirectory, + files.Select(name => (name, blobSize)).ToList(), + cancellationToken); await UploadBlobDirectoryAndVerify( - test.Container, localDirectory, - files, - waitTimeInSec: waitTimeInSec, - options: options); + test.Container, + files.Count, + options: options, + cancellationToken: cancellationToken); } [Ignore("These tests currently take 40+ mins for little additional coverage")] @@ -135,26 +130,29 @@ await UploadBlobDirectoryAndVerify( public async Task LocalToBlockBlobDirectory_LargeSize(long blobSize, int waitTimeInSec) { DataTransferOptions options = new DataTransferOptions(); - List files = new List(); using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); string localDirectory = CreateRandomDirectory(testDirectory.DirectoryPath); await using DisposingContainer test = await GetTestContainerAsync(); - files.Add(await CreateRandomFileAsync(localDirectory, size: blobSize)); - files.Add(await CreateRandomFileAsync(localDirectory, size: blobSize)); - - string openSubfolder = CreateRandomDirectory(localDirectory); - files.Add(await CreateRandomFileAsync(openSubfolder, size: blobSize)); - string lockedSubfolder = CreateRandomDirectory(localDirectory); - files.Add(await CreateRandomFileAsync(lockedSubfolder, size: blobSize)); + List files = new() + { + GetNewBlobName(), + GetNewBlobName(), + $"{GetNewBlobName()}/{GetNewBlobName()}", + $"{GetNewBlobName()}/{GetNewBlobName()}", + }; - // Arrange + CancellationToken cancellationToken = TestHelper.GetTimeoutToken(waitTimeInSec); + await SetupDirectory( + localDirectory, + files.Select(name => (name, blobSize)).ToList(), + cancellationToken); await UploadBlobDirectoryAndVerify( - test.Container, localDirectory, - files, - waitTimeInSec: waitTimeInSec, - options: options); + test.Container, + files.Count, + options: options, + cancellationToken: cancellationToken); } [Test] @@ -168,26 +166,29 @@ public async Task LocalToBlockBlobDirectory_SmallChunks() InitialTransferSize = 100, MaximumTransferChunkSize = 200, }; - List files = new List(); using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); string localDirectory = CreateRandomDirectory(testDirectory.DirectoryPath); await using DisposingContainer test = await GetTestContainerAsync(); - files.Add(await CreateRandomFileAsync(localDirectory, size: blobSize)); - files.Add(await CreateRandomFileAsync(localDirectory, size: blobSize)); - - string openSubfolder = CreateRandomDirectory(localDirectory); - files.Add(await CreateRandomFileAsync(openSubfolder, size: blobSize)); - string lockedSubfolder = CreateRandomDirectory(localDirectory); - files.Add(await CreateRandomFileAsync(lockedSubfolder, size: blobSize)); + List files = new() + { + GetNewBlobName(), + GetNewBlobName(), + $"{GetNewBlobName()}/{GetNewBlobName()}", + $"{GetNewBlobName()}/{GetNewBlobName()}", + }; - // Arrange + CancellationToken cancellationToken = TestHelper.GetTimeoutToken(waitTimeInSec); + await SetupDirectory( + localDirectory, + files.Select(name => (name, blobSize)).ToList(), + cancellationToken); await UploadBlobDirectoryAndVerify( - test.Container, localDirectory, - files, - waitTimeInSec: waitTimeInSec, - options: options); + test.Container, + files.Count, + options: options, + cancellationToken: cancellationToken); } [Test] @@ -207,34 +208,39 @@ public async Task LocalToBlockBlobDirectory_SmallChunks_ManyFiles() InitialTransferSize = 512, MaximumTransferChunkSize = 512, }; - List files = new List(); using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); string localDirectory = CreateRandomDirectory(testDirectory.DirectoryPath); await using DisposingContainer test = await GetTestContainerAsync(); - files.Add(await CreateRandomFileAsync(localDirectory, size: blobSize)); - files.Add(await CreateRandomFileAsync(localDirectory, size: blobSize)); - files.Add(await CreateRandomFileAsync(localDirectory, size: blobSize)); - files.Add(await CreateRandomFileAsync(localDirectory, size: blobSize)); - files.Add(await CreateRandomFileAsync(localDirectory, size: blobSize)); - - string openSubfolder = CreateRandomDirectory(localDirectory); - files.Add(await CreateRandomFileAsync(openSubfolder, size: blobSize)); - files.Add(await CreateRandomFileAsync(openSubfolder, size: blobSize)); - files.Add(await CreateRandomFileAsync(openSubfolder, size: blobSize)); - string openSubfolder2 = CreateRandomDirectory(localDirectory); - files.Add(await CreateRandomFileAsync(openSubfolder2, size: blobSize)); - files.Add(await CreateRandomFileAsync(openSubfolder2, size: blobSize)); + string folder1 = GetNewBlobName(); + string folder2 = GetNewBlobName(); + List files = new() + { + GetNewBlobName(), + GetNewBlobName(), + GetNewBlobName(), + GetNewBlobName(), + GetNewBlobName(), + $"{folder1}/{GetNewBlobName()}", + $"{folder1}/{GetNewBlobName()}", + $"{folder1}/{GetNewBlobName()}", + $"{folder2}/{GetNewBlobName()}", + $"{folder2}/{GetNewBlobName()}", + }; - // Act / Assert + CancellationToken cancellationToken = TestHelper.GetTimeoutToken(waitTimeInSec); + await SetupDirectory( + localDirectory, + files.Select(name => (name, blobSize)).ToList(), + cancellationToken); await UploadBlobDirectoryAndVerify( - test.Container, localDirectory, - files, - waitTimeInSec: waitTimeInSec, + test.Container, + files.Count, transferManagerOptions: transferManagerOptions, - options: options); + options: options, + cancellationToken: cancellationToken); } [Test] @@ -281,24 +287,24 @@ public async Task DirectoryUpload_SingleFile() { // Arrange await using DisposingContainer test = await GetTestContainerAsync(); - - string dirName = GetNewBlobName(); - StorageResourceContainer destinationResource = new BlobStorageResourceContainer(test.Container, new() { BlobDirectoryPrefix = dirName }); - - List files = new List(); using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); string localDirectory = CreateRandomDirectory(testDirectory.DirectoryPath); - StorageResourceContainer sourceResource = new LocalDirectoryStorageResourceContainer(localDirectory); - string openChild = await CreateRandomFileAsync(localDirectory); - files.Add(openChild); + List files = new() + { + GetNewBlobName(), + }; - // Arrange + CancellationToken cancellationToken = TestHelper.GetTimeoutToken(10); + await SetupDirectory( + localDirectory, + files.Select(name => (name, (long)Constants.KB)).ToList(), + cancellationToken); await UploadBlobDirectoryAndVerify( - test.Container, localDirectory, - files, - waitTimeInSec: 10); + test.Container, + files.Count, + cancellationToken: cancellationToken); } [Test] @@ -310,43 +316,30 @@ public async Task DirectoryUpload_ManySubDirectories() using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); string localDirectory = CreateRandomDirectory(testDirectory.DirectoryPath); - string dirName = GetNewBlobName(); - List files = new List(); - - string openSubfolder = CreateRandomDirectory(localDirectory); - string openSubchild = await CreateRandomFileAsync(openSubfolder); - files.Add(openSubchild); - - string openSubfolder2 = CreateRandomDirectory(localDirectory); - string openSubChild2_1 = await CreateRandomFileAsync(openSubfolder2); - string openSubChild2_2 = await CreateRandomFileAsync(openSubfolder2); - string openSubChild2_3 = await CreateRandomFileAsync(openSubfolder2); - files.Add(openSubChild2_1); - files.Add(openSubChild2_2); - files.Add(openSubChild2_3); - - string openSubfolder3 = CreateRandomDirectory(localDirectory); - string openSubChild3_1 = await CreateRandomFileAsync(openSubfolder2); - string openSubChild3_2 = await CreateRandomFileAsync(openSubfolder2); - string openSubChild3_3 = await CreateRandomFileAsync(openSubfolder2); - files.Add(openSubChild3_1); - files.Add(openSubChild3_2); - files.Add(openSubChild3_3); - - string openSubfolder4 = CreateRandomDirectory(localDirectory); - string openSubChild4_1 = await CreateRandomFileAsync(openSubfolder2); - string openSubChild4_2 = await CreateRandomFileAsync(openSubfolder2); - string openSubChild4_3 = await CreateRandomFileAsync(openSubfolder2); - files.Add(openSubChild4_1); - files.Add(openSubChild4_2); - files.Add(openSubChild4_3); + List files = new() + { + GetNewBlobName(), + }; + foreach (string dir in Enumerable.Range(0, 3).Select(_ => GetNewBlobName())) + { + foreach (string blob in Enumerable.Range(0, 3).Select(_ => GetNewBlobName())) + { + files.Add($"{dir}/{blob}"); + } + } + string blobPrefix = GetNewBlobName(); + CancellationToken cancellationToken = TestHelper.GetTimeoutToken(10); + await SetupDirectory( + localDirectory, + files.Select(name => (name, (long)Constants.KB)).ToList(), + cancellationToken); await UploadBlobDirectoryAndVerify( - test.Container, localDirectory, - files, - destinationPrefix: dirName, - waitTimeInSec: 10); + test.Container, + files.Count, + destinationPrefix: blobPrefix, + cancellationToken: cancellationToken); } [Test] @@ -360,24 +353,28 @@ public async Task DirectoryUpload_SubDirectoriesLevels(int level) await using DisposingContainer test = await GetTestContainerAsync(); using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); string localDirectory = CreateRandomDirectory(testDirectory.DirectoryPath); - string dirName = GetNewBlobName(); + string blobName = GetNewBlobName(); List files = new List(); string subfolderName = localDirectory; for (int i = 0; i < level; i++) { - string openSubfolder = CreateRandomDirectory(subfolderName); - files.Add(await CreateRandomFileAsync(openSubfolder)); - subfolderName = openSubfolder; + subfolderName = Path.Combine(subfolderName, GetNewBlobName()); + files.Add(Path.Combine(subfolderName, GetNewBlobName())); } + CancellationToken cancellationToken = TestHelper.GetTimeoutToken(10); + await SetupDirectory( + localDirectory, + files.Select(name => (name, (long)Constants.KB)).ToList(), + cancellationToken); await UploadBlobDirectoryAndVerify( - test.Container, - localDirectory, - files, - destinationPrefix: dirName, - waitTimeInSec: 10); + localDirectory, + test.Container, + files.Count, + destinationPrefix: blobName, + cancellationToken: cancellationToken); } [Test] @@ -404,11 +401,11 @@ public async Task DirectoryUpload_EmptySubDirectories() string openSubfolder4 = CreateRandomDirectory(localDirectory); await UploadBlobDirectoryAndVerify( - test.Container, localDirectory, - files, + test.Container, + expectedTransfers: 0, destinationPrefix: dirName, - waitTimeInSec: 10); + cancellationToken: TestHelper.GetTimeoutToken(10)); } #endregion @@ -425,35 +422,33 @@ public async Task DirectoryUpload_OverwriteTrue() using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); string localDirectory = CreateRandomDirectory(testDirectory.DirectoryPath); - List files = new List(); - string file1 = await CreateRandomFileAsync(localDirectory); - string file2 = await CreateRandomFileAsync(localDirectory); - files.Add(file1); - files.Add(file2); - - string openSubfolder = CreateRandomDirectory(localDirectory); - string file3 = await CreateRandomFileAsync(openSubfolder); - files.Add(file3); - - string lockedSubfolder = CreateRandomDirectory(localDirectory); - string file4 = await CreateRandomFileAsync(lockedSubfolder); - files.Add(file4); + List files = new() + { + GetNewBlobName(), + GetNewBlobName(), + $"{GetNewBlobName()}/{GetNewBlobName()}", + $"{GetNewBlobName()}/{GetNewBlobName()}", + }; DataTransferOptions options = new DataTransferOptions() { CreationPreference = StorageResourceCreationPreference.OverwriteIfExists }; - BlobClient blobClient = test.Container.GetBlobClient(dirName + "/" + file1.Substring(localDirectory.Length + 1).Replace('\\', '/')); - await blobClient.UploadAsync(file1); + BlobClient blobClient = test.Container.GetBlobClient(dirName + "/" + files[0]); + await blobClient.UploadAsync(new BinaryData(GetRandomBuffer(1234))); - // Act + CancellationToken cancellationToken = TestHelper.GetTimeoutToken(10); + await SetupDirectory( + localDirectory, + files.Select(name => (name, (long)Constants.KB)).ToList(), + cancellationToken); await UploadBlobDirectoryAndVerify( - test.Container, localDirectory, - files, + test.Container, + files.Count, destinationPrefix: dirName, - waitTimeInSec: 10, - options: options); + options: options, + cancellationToken: cancellationToken); } [Test] @@ -466,35 +461,34 @@ public async Task DirectoryUpload_OverwriteFalse() string localDirectory = CreateRandomDirectory(testDirectory.DirectoryPath); string dirName = GetNewBlobName(); - List files = new List(); - string file1 = await CreateRandomFileAsync(localDirectory); - string file2 = await CreateRandomFileAsync(localDirectory); - files.Add(file1); - files.Add(file2); - - string openSubfolder = CreateRandomDirectory(localDirectory); - string file3 = await CreateRandomFileAsync(openSubfolder); - files.Add(file3); - - string lockedSubfolder = CreateRandomDirectory(localDirectory); - string file4 = await CreateRandomFileAsync(lockedSubfolder); - files.Add(file4); + List files = new() + { + GetNewBlobName(), + GetNewBlobName(), + $"{GetNewBlobName()}/{GetNewBlobName()}", + $"{GetNewBlobName()}/{GetNewBlobName()}", + }; DataTransferOptions options = new DataTransferOptions() { CreationPreference = StorageResourceCreationPreference.OverwriteIfExists }; - BlobClient blobClient = test.Container.GetBlobClient(dirName + "/" + file1.Substring(localDirectory.Length + 1).Replace('\\', '/')); - await blobClient.UploadAsync(file1); + BlobClient blobClient = test.Container.GetBlobClient(dirName + "/" + files[0]); + await blobClient.UploadAsync(new BinaryData(GetRandomBuffer(1234))); // Act + CancellationToken cancellationToken = TestHelper.GetTimeoutToken(10); + await SetupDirectory( + localDirectory, + files.Select(name => (name, (long)Constants.KB)).ToList(), + cancellationToken); await UploadBlobDirectoryAndVerify( - test.Container, localDirectory, - files, + test.Container, + files.Count, destinationPrefix: dirName, - waitTimeInSec: 10, - options: options); + options: options, + cancellationToken: cancellationToken); } [Test]