diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/assets.json b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/assets.json index b07f03aebb9a..5009ed728418 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/assets.json +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.DataMovement.Files.Shares", - "Tag": "net/storage/Azure.Storage.DataMovement.Files.Shares_876c524ce2" + "Tag": "net/storage/Azure.Storage.DataMovement.Files.Shares_4e52c7c39c" } diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj index b0a6b1f0a941..2b89825eeecd 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj @@ -44,6 +44,7 @@ + diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferDownloadTests.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferDownloadTests.cs index 5f5734ec28bb..011856220eb5 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferDownloadTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferDownloadTests.cs @@ -61,7 +61,7 @@ protected override async Task CreateObjectClientAsync( } protected override TransferValidator.ListFilesAsync GetSourceLister(ShareClient container, string prefix) - => TransferValidator.GetShareFilesLister(container, prefix); + => TransferValidator.GetShareFileLister(container.GetDirectoryClient(prefix)); protected override StorageResourceContainer GetStorageResourceContainer(ShareClient container, string directoryPath) { diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferUploadTests.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferUploadTests.cs new file mode 100644 index 000000000000..993af43dd6d9 --- /dev/null +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferUploadTests.cs @@ -0,0 +1,101 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Azure.Storage.DataMovement.Tests; +using Azure.Storage.Files.Shares; +using Azure.Storage.Files.Shares.Tests; +using Azure.Storage.Test.Shared; + +namespace Azure.Storage.DataMovement.Files.Shares.Tests +{ + [ShareClientTestFixture(true)] + [ShareClientTestFixture(false)] + internal class ShareDirectoryStartTransferUploadTests : StartTransferUploadDirectoryTestBase< + ShareServiceClient, + ShareDirectoryClient, + ShareFileClient, + ShareClientOptions, + StorageTestEnvironment> + { + /// + /// A but exposes a directory client within that share. + /// Still cleans up the whole share. Helpful for parameterizing tests to use a root + /// directory vs a subdir. + /// + private class DisposingShareDirectory : IDisposingContainer + { + private readonly DisposingShare _disposingShare; + public ShareDirectoryClient Container { get; } + + public DisposingShareDirectory(DisposingShare disposingShare, ShareDirectoryClient dirClient) + { + _disposingShare = disposingShare; + Container = dirClient; + } + + public async ValueTask DisposeAsync() + { + if (_disposingShare != default) + { + await _disposingShare.DisposeAsync(); + } + } + } + + public bool UseNonRootDirectory { get; } + + public ShareDirectoryStartTransferUploadTests(bool async, ShareClientOptions.ServiceVersion serviceVersion, bool useNonRootDirectory) + : base(async, null /* RecordedTestMode.Record /* to re-record */) + { + ClientBuilder = ClientBuilderExtensions.GetNewShareClientBuilder(Tenants, serviceVersion); + UseNonRootDirectory = useNonRootDirectory; + } + + protected override async Task> GetDisposingContainerAsync(ShareServiceClient service = null, string containerName = null) + { + DisposingShare disposingShare = await ClientBuilder.GetTestShareAsync(service, containerName); + ShareDirectoryClient directoryClient = disposingShare.Container.GetRootDirectoryClient(); + if (UseNonRootDirectory) + { + foreach (var _ in Enumerable.Range(0, 2)) + { + directoryClient = directoryClient.GetSubdirectoryClient(GetNewObjectName()); + await directoryClient.CreateAsync(); + } + } + return new DisposingShareDirectory(disposingShare, directoryClient); + } + + protected override StorageResourceContainer GetStorageResourceContainer(ShareDirectoryClient containerClient) + { + return new ShareDirectoryStorageResourceContainer(containerClient, null); + } + + protected override TransferValidator.ListFilesAsync GetStorageResourceLister(ShareDirectoryClient containerClient) + { + return TransferValidator.GetShareFileLister(containerClient); + } + + protected override async Task InitializeDestinationDataAsync(ShareDirectoryClient containerClient, List<(string FilePath, long Size)> fileSizes, CancellationToken cancellationToken) + { + foreach ((string filePath, long size) in fileSizes) + { + ShareDirectoryClient directory = containerClient; + + string[] pathSegments = filePath.Split('/'); + foreach (string pathSegment in pathSegments.Take(pathSegments.Length - 1)) + { + directory = directory.GetSubdirectoryClient(pathSegment); + await directory.CreateIfNotExistsAsync(cancellationToken: cancellationToken); + } + ShareFileClient file = directory.GetFileClient(pathSegments.Last()); + await file.CreateAsync(size, cancellationToken: cancellationToken); + await file.UploadAsync(await CreateLimitedMemoryStream(size), cancellationToken: cancellationToken); + } + } + } +} diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/TransferValidator.Shares.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/TransferValidator.Shares.cs index 8c6917ae007c..4366f075e64c 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/TransferValidator.Shares.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/TransferValidator.Shares.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +using System; using System.Collections.Generic; using System.IO; using System.Threading; @@ -12,13 +13,13 @@ namespace Azure.Storage.DataMovement.Tests { public partial class TransferValidator { - public class ShareResourceEnumerationItem : IResourceEnumerationItem + public class ShareFileResourceEnumerationItem : IResourceEnumerationItem { private readonly ShareFileClient _client; public string RelativePath { get; } - public ShareResourceEnumerationItem(ShareFileClient client, string relativePath) + public ShareFileResourceEnumerationItem(ShareFileClient client, string relativePath) { _client = client; RelativePath = relativePath; @@ -30,66 +31,39 @@ public async Task OpenReadAsync(CancellationToken cancellationToken) } } - public static ListFilesAsync GetShareFilesLister(ShareClient container, string prefix) + public static ListFilesAsync GetShareFileLister(ShareDirectoryClient container) { - async Task> ListFiles(CancellationToken cancellationToken) + async Task> ListFilesRecursive(ShareDirectoryClient dir, CancellationToken cancellationToken) { List result = new(); - ShareDirectoryClient directory = string.IsNullOrEmpty(prefix) ? - container.GetRootDirectoryClient() : - container.GetDirectoryClient(prefix); - - Queue toScan = new(); - toScan.Enqueue(directory); - - while (toScan.Count > 0) + await foreach (ShareFileItem fileItem in dir.GetFilesAndDirectoriesAsync(cancellationToken: cancellationToken)) { - ShareDirectoryClient current = toScan.Dequeue(); - await foreach (ShareFileItem item in current.GetFilesAndDirectoriesAsync( - cancellationToken: cancellationToken).ConfigureAwait(false)) + if (fileItem.IsDirectory) + { + result.AddRange(await ListFilesRecursive(dir.GetSubdirectoryClient(fileItem.Name), cancellationToken)); + } + else { - if (item.IsDirectory) - { - ShareDirectoryClient subdir = current.GetSubdirectoryClient(item.Name); - toScan.Enqueue(subdir); - } - else - { - string relativePath = ""; - if (string.IsNullOrEmpty(current.Path)) - { - relativePath = item.Name; - } - else if (string.IsNullOrEmpty(prefix)) - { - relativePath = string.Join("/", current.Path, item.Name); - } - else - { - relativePath = - prefix != current.Name ? - string.Join("/", current.Path.Substring(prefix.Length + 1), item.Name) : - item.Name; - } - result.Add(new ShareResourceEnumerationItem(current.GetFileClient(item.Name), relativePath)); - } + ShareFileClient fileClient = dir.GetFileClient(fileItem.Name); + result.Add(new ShareFileResourceEnumerationItem( + fileClient, fileClient.Path.Substring(container.Path.Length).Trim('/'))); } } return result; } - return ListFiles; + return (cancellationToken) => ListFilesRecursive(container, cancellationToken); } - public static ListFilesAsync GetFileListerSingle(ShareFileClient file, string relativePath) + public static ListFilesAsync GetShareFileListerSingle(ShareFileClient file, string relativePath) { - Task> ListFiles(CancellationToken cancellationToken) + Task> ListFile(CancellationToken cancellationToken) { return Task.FromResult(new List { - new ShareResourceEnumerationItem(file, relativePath) + new ShareFileResourceEnumerationItem(file, relativePath) }); } - return ListFiles; + return ListFile; } } } diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferUploadDirectoryTestBase.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferUploadDirectoryTestBase.cs new file mode 100644 index 000000000000..0fdb9b1bd0a5 --- /dev/null +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferUploadDirectoryTestBase.cs @@ -0,0 +1,485 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core; +using Azure.Core.TestFramework; +using Azure.Storage.Test; +using Azure.Storage.Test.Shared; +using NUnit.Framework; + +namespace Azure.Storage.DataMovement.Tests +{ + public abstract class StartTransferUploadDirectoryTestBase< + TServiceClient, + TContainerClient, + TObjectClient, + TClientOptions, + TEnvironment> + : StorageTestBase + where TServiceClient : class + where TContainerClient : class + where TObjectClient : class + where TClientOptions : ClientOptions + where TEnvironment : StorageTestEnvironment, new() + { + private const long DefaultObjectSize = Constants.KB; + + //private readonly string _generatedResourceNamePrefix; + //private readonly string _expectedOverwriteExceptionMessage; + + public ClientBuilder ClientBuilder { get; protected set; } + + public LocalFilesStorageResourceProvider LocalResourceProvider { get; } = new(); + + public StartTransferUploadDirectoryTestBase(bool async, RecordedTestMode? mode = null) + : base(async, mode) + { } + + protected string GetNewObjectName(int? maxChars = 8) + { + string result = ClientBuilder.Recording.Random.NewGuid().ToString(); + return maxChars < result.Length ? result.Substring(0, maxChars.Value) : result; + } + + #region Service-Specific Implementations + /// + /// Gets a service-specific disposing container for use with tests in this class. + /// + /// Optionally specified service client to get container from. + /// Optional container name specification. + protected abstract Task> GetDisposingContainerAsync( + TServiceClient service = default, + string containerName = default); + + /// + /// Initializes data at the destination container. + /// + /// + protected abstract Task InitializeDestinationDataAsync( + TContainerClient containerClient, + List<(string FilePath, long Size)> fileSizes, + CancellationToken cancellationToken); + + /// + /// Gets the specific storage resource from the given TObjectClient + /// e.g. ShareFileClient to a ShareFileStorageResource, BlockBlobClient to a BlockBlobStorageResource. + /// + /// The object client to create the storage resource object. + /// + protected abstract StorageResourceContainer GetStorageResourceContainer(TContainerClient containerClient); + + /// + /// Gets the appropriate delegate for listing through and validating the state of the destination container. + /// + /// + /// + /// + protected abstract TransferValidator.ListFilesAsync GetStorageResourceLister(TContainerClient containerClient); + #endregion + + #region Test Helpers + private async Task SetupDirectoryAsync( + string directoryPath, + List<(string FilePath, long Size)> fileSizes, + CancellationToken cancellationToken) + { + foreach ((string filePath, long size) in fileSizes) + { + string currRelPath = ""; + string[] pathSegments = filePath.Split('/', '\\'); + if (pathSegments.Length < 1) + { + continue; + } + foreach (string directoryName in pathSegments.Take(pathSegments.Length - 1)) + { + currRelPath = string.Join(Path.DirectorySeparatorChar.ToString(), currRelPath, directoryName).Trim(Path.DirectorySeparatorChar); + string currAbsPath = Path.Combine(directoryPath, currRelPath); + if (!Directory.Exists(currAbsPath)) + { + Directory.CreateDirectory(currAbsPath); + } + } + + currRelPath = string.Join(Path.DirectorySeparatorChar.ToString(), currRelPath, pathSegments.Last()).Trim(Path.DirectorySeparatorChar); + if (size < 0) + { + Directory.CreateDirectory(Path.Combine(directoryPath, currRelPath)); + } + else + { + using FileStream fs = File.OpenWrite(Path.Combine(directoryPath, currRelPath)); + using Stream data = await CreateLimitedMemoryStream(size); + await data.CopyToAsync(fs, bufferSize: 4 * Constants.KB, cancellationToken); + } + } + } + + /// + /// Upload and verify the contents of the directory + /// + /// By default in this function an event arguement will be added to the options event handler + /// to detect when the upload has finished. + /// + private async Task UploadDirectoryAndVerifyAsync( + string sourceLocalDirectoryPath, + TContainerClient destinationContainer, + int expectedTransfers, + TransferManagerOptions transferManagerOptions = default, + DataTransferOptions options = default, + CancellationToken cancellationToken = default) + { + // Set transfer options + options ??= new DataTransferOptions(); + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + + transferManagerOptions ??= new TransferManagerOptions() + { + ErrorHandling = DataTransferErrorMode.ContinueOnFailure + }; + + StorageResourceContainer sourceResource = LocalResourceProvider.FromDirectory(sourceLocalDirectoryPath); + StorageResourceContainer destinationResource = GetStorageResourceContainer(destinationContainer); + + await new TransferValidator() + { + TransferManager = new(transferManagerOptions) + }.TransferAndVerifyAsync( + sourceResource, + destinationResource, + TransferValidator.GetLocalFileLister(sourceLocalDirectoryPath), + GetStorageResourceLister(destinationContainer), + expectedTransfers, + options, + cancellationToken); + } + #endregion + + [RecordedTest] + [TestCase(Constants.KB, 2)] + [TestCase(12345, 2)] + public async Task Upload(long objectSize, int waitTimeInSec) + { + // Arrange + using DisposingLocalDirectory disposingLocalDirectory = DisposingLocalDirectory.GetTestDirectory(); + await using IDisposingContainer test = await GetDisposingContainerAsync(); + + List files = new() + { + GetNewObjectName(), + GetNewObjectName(), + }; + Console.WriteLine($"files: {string.Join(", ", files)}"); + + CancellationToken cancellationToken = TestHelper.GetTimeoutToken(waitTimeInSec); + await SetupDirectoryAsync( + disposingLocalDirectory.DirectoryPath, + files.Select(path => (path, objectSize)).ToList(), + cancellationToken); + await UploadDirectoryAndVerifyAsync( + disposingLocalDirectory.DirectoryPath, + test.Container, + expectedTransfers: files.Count, + cancellationToken: cancellationToken); + } + + [RecordedTest] + [TestCase(DataTransferErrorMode.ContinueOnFailure)] + [TestCase(DataTransferErrorMode.StopOnAnyFailure)] + public async Task UploadFailIfExists(DataTransferErrorMode errorMode) + { + const int waitTimeInSec = 5; + const int preexistingFileCount = 2; + const int skipCount = 1; + const int totalFileCount = skipCount + preexistingFileCount + 1; + // Arrange + using DisposingLocalDirectory disposingLocalDirectory = DisposingLocalDirectory.GetTestDirectory(); + await using IDisposingContainer test = await GetDisposingContainerAsync(); + + List files = new(); + foreach (var _ in Enumerable.Range(0, totalFileCount)) + { + files.Add(GetNewObjectName()); + } + + CancellationToken cancellationToken = TestHelper.GetTimeoutToken(waitTimeInSec); + await InitializeDestinationDataAsync( + test.Container, + files.Skip(skipCount).Take(preexistingFileCount).Select(path => (path, DefaultObjectSize)).ToList(), + cancellationToken); + await SetupDirectoryAsync( + disposingLocalDirectory.DirectoryPath, + files.Select(path => (path, DefaultObjectSize)).ToList(), + cancellationToken); + + DataTransferOptions options = new() + { + CreationPreference = StorageResourceCreationPreference.FailIfExists + }; + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + TransferManagerOptions transferManagerOptions = new() + { + ErrorHandling = DataTransferErrorMode.ContinueOnFailure + }; + + StorageResourceContainer sourceResource = LocalResourceProvider.FromDirectory(disposingLocalDirectory.DirectoryPath); + StorageResourceContainer destinationResource = GetStorageResourceContainer(test.Container); + DataTransfer transfer = await new TransferManager(transferManagerOptions) + .StartTransferAsync(sourceResource, destinationResource, options, cancellationToken); + await transfer.WaitForCompletionAsync(cancellationToken); + + // check if expected files exist, but not necessarily for contents + if (errorMode == DataTransferErrorMode.ContinueOnFailure) + { + await testEventsRaised.AssertContainerCompletedWithFailedCheckContinue(preexistingFileCount); + + // Verify all files exist, meaning files without conflict were transferred. + List localFiles = (await TransferValidator.GetLocalFileLister(disposingLocalDirectory.DirectoryPath) + .Invoke(cancellationToken)) + .Select(item => item.RelativePath) + .ToList(); + List destinationObjects = (await GetStorageResourceLister(test.Container) + .Invoke(cancellationToken)) + .Select(item => item.RelativePath) + .ToList(); + Assert.That(localFiles, Is.EquivalentTo(destinationObjects)); + } + else if (errorMode == DataTransferErrorMode.StopOnAnyFailure) + { + Assert.That(transfer.TransferStatus.HasFailedItems, Is.True); + } + } + + [RecordedTest] + [Test] + public async Task UploadSkipIfExists() + { + const int waitTimeInSec = 5; + const int preexistingFileCount = 2; + const int skipCount = 1; + const int totalFileCount = skipCount + preexistingFileCount + 1; + // Arrange + using DisposingLocalDirectory disposingLocalDirectory = DisposingLocalDirectory.GetTestDirectory(); + await using IDisposingContainer test = await GetDisposingContainerAsync(); + + List files = new(); + foreach (var _ in Enumerable.Range(0, totalFileCount)) + { + files.Add(GetNewObjectName()); + } + + CancellationToken cancellationToken = TestHelper.GetTimeoutToken(waitTimeInSec); + await InitializeDestinationDataAsync( + test.Container, + files.Skip(skipCount).Take(preexistingFileCount).Select(path => (path, DefaultObjectSize)).ToList(), + cancellationToken); + await SetupDirectoryAsync( + disposingLocalDirectory.DirectoryPath, + files.Select(path => (path, DefaultObjectSize)).ToList(), + cancellationToken); + + DataTransferOptions options = new() + { + CreationPreference = StorageResourceCreationPreference.SkipIfExists + }; + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + TransferManagerOptions transferManagerOptions = new() + { + ErrorHandling = DataTransferErrorMode.ContinueOnFailure + }; + + StorageResourceContainer sourceResource = LocalResourceProvider.FromDirectory(disposingLocalDirectory.DirectoryPath); + StorageResourceContainer destinationResource = GetStorageResourceContainer(test.Container); + DataTransfer transfer = await new TransferManager(transferManagerOptions) + .StartTransferAsync(sourceResource, destinationResource, options, cancellationToken); + await transfer.WaitForCompletionAsync(cancellationToken); + + // check if expected files exist, but not necessarily for contents + await testEventsRaised.AssertContainerCompletedWithSkippedCheck(preexistingFileCount); + + // Verify all files exist, meaning files without conflict were transferred. + List localFiles = (await TransferValidator.GetLocalFileLister(disposingLocalDirectory.DirectoryPath) + .Invoke(cancellationToken)) + .Select(item => item.RelativePath) + .ToList(); + List destinationObjects = (await GetStorageResourceLister(test.Container) + .Invoke(cancellationToken)) + .Select(item => item.RelativePath) + .ToList(); + Assert.That(localFiles, Is.EquivalentTo(destinationObjects)); + } + + [RecordedTest] + [Test] + public async Task UploadOverwriteIfExists() + { + const int waitTimeInSec = 3; + // Arrange + using DisposingLocalDirectory disposingLocalDirectory = DisposingLocalDirectory.GetTestDirectory(); + await using IDisposingContainer test = await GetDisposingContainerAsync(); + + List files = new() + { + GetNewObjectName(), + GetNewObjectName(), + }; + + DataTransferOptions options = new() + { + CreationPreference = StorageResourceCreationPreference.OverwriteIfExists + }; + CancellationToken cancellationToken = TestHelper.GetTimeoutToken(waitTimeInSec); + await InitializeDestinationDataAsync( + test.Container, + files.Take(1).Select(path => (path, DefaultObjectSize)).ToList(), + cancellationToken); + await SetupDirectoryAsync( + disposingLocalDirectory.DirectoryPath, + files.Select(path => (path, DefaultObjectSize)).ToList(), + cancellationToken); + await UploadDirectoryAndVerifyAsync( + disposingLocalDirectory.DirectoryPath, + test.Container, + expectedTransfers: files.Count, + options: options, + cancellationToken: cancellationToken); + } + + [RecordedTest] + [TestCase(Constants.KB, Constants.KB/4, 2)] + [TestCase(10 * Constants.KB, 4 * Constants.KB, 5)] + [TestCase(Constants.KB, 97, 2)] + public async Task UploadSmallChunks(long objectSize, long chunkSize, int waitTimeInSec) + { + // Arrange + using DisposingLocalDirectory disposingLocalDirectory = DisposingLocalDirectory.GetTestDirectory(); + await using IDisposingContainer test = await GetDisposingContainerAsync(); + + List files = new() + { + GetNewObjectName(), + GetNewObjectName(), + }; + + DataTransferOptions options = new() + { + InitialTransferSize = chunkSize, + MaximumTransferChunkSize = chunkSize, + }; + + CancellationToken cancellationToken = TestHelper.GetTimeoutToken(waitTimeInSec); + await SetupDirectoryAsync( + disposingLocalDirectory.DirectoryPath, + files.Select(path => (path, objectSize)).ToList(), + cancellationToken); + await UploadDirectoryAndVerifyAsync( + disposingLocalDirectory.DirectoryPath, + test.Container, + expectedTransfers: files.Count, + cancellationToken: cancellationToken); + } + + [RecordedTest] + [TestCase(1)] + [TestCase(5)] + public async Task UploadEmpty(int folderDepth) + { + const int waitTimeInSec = 10; + // Arrange + using DisposingLocalDirectory disposingLocalDirectory = DisposingLocalDirectory.GetTestDirectory(); + await using IDisposingContainer test = await GetDisposingContainerAsync(); + + void BuildFolders(string path, int depth) + { + if (depth < 1) + { + return; + } + for (int i = 0; i < 2; i++) + { + string subDirPath = Path.Combine(path, GetNewObjectName()); + Directory.CreateDirectory(subDirPath); + BuildFolders(subDirPath, depth - 1); + } + } + BuildFolders(disposingLocalDirectory.DirectoryPath, folderDepth); + + CancellationToken cancellationToken = TestHelper.GetTimeoutToken(waitTimeInSec); + await UploadDirectoryAndVerifyAsync( + disposingLocalDirectory.DirectoryPath, + test.Container, + expectedTransfers: 0, + cancellationToken: cancellationToken); + } + + [Ignore("Times out on linux/mac, currently unsure why.")] + [RecordedTest] + [TestCase(1, 5)] + [TestCase(3, 10)] + public async Task UploadManySubdirectories(int folderDepth, int waitTimeInSec) + { + // Arrange + using DisposingLocalDirectory disposingLocalDirectory = DisposingLocalDirectory.GetTestDirectory(); + await using IDisposingContainer test = await GetDisposingContainerAsync(); + + List files = new(); + void BuildFilePaths(string path, int depth) + { + if (depth < 1) + { + files.Add(Path.Combine(path, GetNewObjectName())); + return; + } + for (int i = 0; i < 2; i++) + { + BuildFilePaths(Path.Combine(path, GetNewObjectName()), depth - 1); + } + } + BuildFilePaths(disposingLocalDirectory.DirectoryPath, folderDepth); + + CancellationToken cancellationToken = TestHelper.GetTimeoutToken(waitTimeInSec); + await SetupDirectoryAsync( + disposingLocalDirectory.DirectoryPath, + files.Select(path => (path, DefaultObjectSize)).ToList(), + cancellationToken); + await UploadDirectoryAndVerifyAsync( + disposingLocalDirectory.DirectoryPath, + test.Container, + expectedTransfers: 1 << folderDepth, + cancellationToken: cancellationToken); + } + + [RecordedTest] + [TestCase(1)] + [TestCase(5)] + public async Task UploadSingleFile(int folderDepth) + { + const int waitTimeInSec = 5; + // Arrange + using DisposingLocalDirectory disposingLocalDirectory = DisposingLocalDirectory.GetTestDirectory(); + await using IDisposingContainer test = await GetDisposingContainerAsync(); + + List files = new() + { + string.Join(Path.DirectorySeparatorChar.ToString(), Enumerable.Range(0, folderDepth).Select(_ => GetNewObjectName()).ToList()) + }; + + CancellationToken cancellationToken = TestHelper.GetTimeoutToken(waitTimeInSec); + await SetupDirectoryAsync( + disposingLocalDirectory.DirectoryPath, + files.Select(path => (path, DefaultObjectSize)).ToList(), + cancellationToken); + await UploadDirectoryAndVerifyAsync( + disposingLocalDirectory.DirectoryPath, + test.Container, + expectedTransfers: 1, + cancellationToken: cancellationToken); + } + } +} diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/TransferValidator.Local.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/TransferValidator.Local.cs index 6316d21e193b..457b7f65431d 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/TransferValidator.Local.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/TransferValidator.Local.cs @@ -44,7 +44,7 @@ Task> ListFiles(CancellationToken cancellationTok } foreach (string filePath in Directory.GetFiles(workingDir)) { - result.Add(new LocalFileResourceEnumerationItem(filePath, filePath.Substring(directoryPath.Length))); + result.Add(new LocalFileResourceEnumerationItem(filePath, filePath.Substring(directoryPath.Length + 1))); } } return Task.FromResult(result); diff --git a/sdk/storage/Azure.Storage.Files.Shares/tests/ShareClientTestFixtureAttribute.cs b/sdk/storage/Azure.Storage.Files.Shares/tests/ShareClientTestFixtureAttribute.cs index 69ebba530e77..89a235486192 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/tests/ShareClientTestFixtureAttribute.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/tests/ShareClientTestFixtureAttribute.cs @@ -1,36 +1,42 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +using System; using Azure.Core.TestFramework; namespace Azure.Storage.Files.Shares.Tests { + [AttributeUsage(AttributeTargets.Class, AllowMultiple = true, Inherited = true)] public class ShareClientTestFixtureAttribute : ClientTestFixtureAttribute { - public ShareClientTestFixtureAttribute() + public ShareClientTestFixtureAttribute(params object[] additionalParameters) : base( - ShareClientOptions.ServiceVersion.V2019_02_02, - ShareClientOptions.ServiceVersion.V2019_07_07, - ShareClientOptions.ServiceVersion.V2019_12_12, - ShareClientOptions.ServiceVersion.V2020_02_10, - ShareClientOptions.ServiceVersion.V2020_04_08, - ShareClientOptions.ServiceVersion.V2020_06_12, - ShareClientOptions.ServiceVersion.V2020_08_04, - ShareClientOptions.ServiceVersion.V2020_10_02, - ShareClientOptions.ServiceVersion.V2020_12_06, - ShareClientOptions.ServiceVersion.V2021_02_12, - ShareClientOptions.ServiceVersion.V2021_04_10, - ShareClientOptions.ServiceVersion.V2021_06_08, - ShareClientOptions.ServiceVersion.V2021_08_06, - ShareClientOptions.ServiceVersion.V2021_10_04, - ShareClientOptions.ServiceVersion.V2021_12_02, - ShareClientOptions.ServiceVersion.V2022_11_02, - ShareClientOptions.ServiceVersion.V2023_01_03, - ShareClientOptions.ServiceVersion.V2023_05_03, - ShareClientOptions.ServiceVersion.V2023_08_03, - ShareClientOptions.ServiceVersion.V2023_11_03, - StorageVersionExtensions.LatestVersion, - StorageVersionExtensions.MaxVersion) + serviceVersions: new object[] + { + ShareClientOptions.ServiceVersion.V2019_02_02, + ShareClientOptions.ServiceVersion.V2019_07_07, + ShareClientOptions.ServiceVersion.V2019_12_12, + ShareClientOptions.ServiceVersion.V2020_02_10, + ShareClientOptions.ServiceVersion.V2020_04_08, + ShareClientOptions.ServiceVersion.V2020_06_12, + ShareClientOptions.ServiceVersion.V2020_08_04, + ShareClientOptions.ServiceVersion.V2020_10_02, + ShareClientOptions.ServiceVersion.V2020_12_06, + ShareClientOptions.ServiceVersion.V2021_02_12, + ShareClientOptions.ServiceVersion.V2021_04_10, + ShareClientOptions.ServiceVersion.V2021_06_08, + ShareClientOptions.ServiceVersion.V2021_08_06, + ShareClientOptions.ServiceVersion.V2021_10_04, + ShareClientOptions.ServiceVersion.V2021_12_02, + ShareClientOptions.ServiceVersion.V2022_11_02, + ShareClientOptions.ServiceVersion.V2023_01_03, + ShareClientOptions.ServiceVersion.V2023_05_03, + ShareClientOptions.ServiceVersion.V2023_08_03, + ShareClientOptions.ServiceVersion.V2023_11_03, + StorageVersionExtensions.LatestVersion, + StorageVersionExtensions.MaxVersion + }, + additionalParameters: additionalParameters) { RecordingServiceVersion = StorageVersionExtensions.MaxVersion; LiveServiceVersions = new object[] { StorageVersionExtensions.LatestVersion, };