From 81930414fc5bd7ef09cd4bc00c8172f1b4c98f46 Mon Sep 17 00:00:00 2001 From: Amanda Nguyen Date: Thu, 25 Sep 2025 15:01:03 -0700 Subject: [PATCH 1/8] Set ChannelProcessor process to null in cleanup; exception handler in chunkhandlers will log unexpected exceptions; remove Transfermanager TransferInternalState reference after transfer completes; Dispose TransferManager in ClientExtensions --- .../src/BlobContainerClientExtensions.cs | 40 +++++++++-------- .../src/ShareDirectoryClientExtensions.cs | 44 ++++++++++--------- .../src/ChannelProcessing.cs | 3 ++ .../src/CommitChunkHandler.cs | 20 +++++++-- .../src/DataMovementEventSource.cs | 7 +++ .../src/DownloadChunkHandler.cs | 16 ++++++- .../src/TransferInternalState.cs | 3 ++ 7 files changed, 91 insertions(+), 42 deletions(-) diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobContainerClientExtensions.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobContainerClientExtensions.cs index bcacd2629a37..467b47365159 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobContainerClientExtensions.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobContainerClientExtensions.cs @@ -14,8 +14,6 @@ namespace Azure.Storage.Blobs /// public static class BlobContainerClientExtensions { - private static Lazy s_defaultTransferManager = new Lazy(() => new TransferManager(default)); - /// /// Uploads the entire contents of local directory to the blob container. /// @@ -89,18 +87,20 @@ public static async Task UploadDirectoryAsync( StorageResource localDirectory = LocalFilesStorageResourceProvider.FromDirectory(localDirectoryPath); StorageResource blobDirectory = BlobsStorageResourceProvider.FromClient(client, options?.BlobContainerOptions); - TransferOperation transfer = await s_defaultTransferManager.Value.StartTransferAsync( - localDirectory, - blobDirectory, - options?.TransferOptions, - cancellationToken).ConfigureAwait(false); - - if (waitUntil == WaitUntil.Completed) + await using (TransferManager transferManager = new TransferManager()) { - await transfer.WaitForCompletionAsync(cancellationToken).ConfigureAwait(false); - } + TransferOperation transfer = await transferManager.StartTransferAsync( + localDirectory, + blobDirectory, + options?.TransferOptions, + cancellationToken).ConfigureAwait(false); - return transfer; + if (waitUntil == WaitUntil.Completed) + { + await transfer.WaitForCompletionAsync(cancellationToken).ConfigureAwait(false); + } + return transfer; + } } /// @@ -176,18 +176,22 @@ public static async Task DownloadToDirectoryAsync( StorageResource localDirectory = LocalFilesStorageResourceProvider.FromDirectory(localDirectoryPath); StorageResource blobDirectory = BlobsStorageResourceProvider.FromClient(client, options?.BlobContainerOptions); - TransferOperation transfer = await s_defaultTransferManager.Value.StartTransferAsync( + // Create a new TransferManager for each operation + await using (TransferManager transferManager = new TransferManager()) + { + TransferOperation transfer = await transferManager.StartTransferAsync( blobDirectory, localDirectory, options?.TransferOptions, cancellationToken).ConfigureAwait(false); - if (waitUntil == WaitUntil.Completed) - { - await transfer.WaitForCompletionAsync(cancellationToken).ConfigureAwait(false); - } + if (waitUntil == WaitUntil.Completed) + { + await transfer.WaitForCompletionAsync(cancellationToken).ConfigureAwait(false); + } - return transfer; + return transfer; + } } } } diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryClientExtensions.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryClientExtensions.cs index 7c80d5b0245c..b4100b914365 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryClientExtensions.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryClientExtensions.cs @@ -14,8 +14,6 @@ namespace Azure.Storage.Files.Shares /// public static class ShareDirectoryClientExtensions { - private static Lazy s_defaultTransferManager = new(() => new TransferManager(default)); - /// /// Uploads the entire contents of local directory to the share directory. /// @@ -48,17 +46,20 @@ public static async Task UploadDirectoryAsync( StorageResource localDirectory = LocalFilesStorageResourceProvider.FromDirectory(localDirectoryPath); StorageResource shareDirectory = ShareFilesStorageResourceProvider.FromClient(client, options?.ShareDirectoryOptions); - TransferOperation trasnfer = await s_defaultTransferManager.Value.StartTransferAsync( - localDirectory, - shareDirectory, - options?.TransferOptions, - cancellationToken).ConfigureAwait(false); - if (waitUntil == WaitUntil.Completed) + await using (TransferManager transferManager = new TransferManager()) { - await trasnfer.WaitForCompletionAsync(cancellationToken).ConfigureAwait(false); - } + TransferOperation transfer = await transferManager.StartTransferAsync( + localDirectory, + shareDirectory, + options?.TransferOptions, + cancellationToken).ConfigureAwait(false); + if (waitUntil == WaitUntil.Completed) + { + await transfer.WaitForCompletionAsync(cancellationToken).ConfigureAwait(false); + } - return trasnfer; + return transfer; + } } /// @@ -93,17 +94,20 @@ public static async Task DownloadToDirectoryAsync( StorageResource localDirectory = LocalFilesStorageResourceProvider.FromDirectory(localDirectoryPath); StorageResource shareDirectory = ShareFilesStorageResourceProvider.FromClient(client, options?.ShareDirectoryOptions); - TransferOperation trasnfer = await s_defaultTransferManager.Value.StartTransferAsync( - shareDirectory, - localDirectory, - options?.TransferOptions, - cancellationToken).ConfigureAwait(false); - if (waitUntil == WaitUntil.Completed) + await using (TransferManager transferManager = new TransferManager()) { - await trasnfer.WaitForCompletionAsync(cancellationToken).ConfigureAwait(false); - } + TransferOperation transfer = await transferManager.StartTransferAsync( + shareDirectory, + localDirectory, + options?.TransferOptions, + cancellationToken).ConfigureAwait(false); + if (waitUntil == WaitUntil.Completed) + { + await transfer.WaitForCompletionAsync(cancellationToken).ConfigureAwait(false); + } - return trasnfer; + return transfer; + } } } } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/ChannelProcessing.cs b/sdk/storage/Azure.Storage.DataMovement/src/ChannelProcessing.cs index 172fa7e6b847..6a0b165aa8e7 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/ChannelProcessing.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/ChannelProcessing.cs @@ -91,6 +91,9 @@ public async Task CleanUpAsync() { _channel.Writer.TryComplete(); await _processorTaskCompletionSource.Task.ConfigureAwait(false); + + // Null out the Process delegate to release references + Interlocked.Exchange(ref _process, null); } protected abstract ValueTask NotifyOfPendingItemProcessing(); diff --git a/sdk/storage/Azure.Storage.DataMovement/src/CommitChunkHandler.cs b/sdk/storage/Azure.Storage.DataMovement/src/CommitChunkHandler.cs index fc5ec066de68..34a4d2dd560d 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/CommitChunkHandler.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/CommitChunkHandler.cs @@ -83,10 +83,10 @@ public CommitChunkHandler( _isChunkHandlerRunning = true; } - public Task CleanUpAsync() + public async Task CleanUpAsync() { _isChunkHandlerRunning = false; - return _stageChunkProcessor.CleanUpAsync(); + await _stageChunkProcessor.CleanUpAsync().ConfigureAwait(false); } public async ValueTask QueueChunkAsync(QueueStageChunkArgs args, CancellationToken cancellationToken = default) @@ -137,7 +137,21 @@ await _queuePutBlockTask( if (_isChunkHandlerRunning) { // This will trigger the job part to call Dispose on this object - _ = Task.Run(() => _invokeFailedEventHandler(ex)); + _ = Task.Run(async () => + { + try + { + await _invokeFailedEventHandler(ex).ConfigureAwait(false); + } + catch + { + // Log and swallow any exceptions to prevent crashing the process + DataMovementEventSource.Singleton + .UnexpectedTransferFailed( + nameof(CommitChunkHandler), + ex.ToString()); + } + }); } } } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/DataMovementEventSource.cs b/sdk/storage/Azure.Storage.DataMovement/src/DataMovementEventSource.cs index 6c65c77b230b..6acdcc982be3 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/DataMovementEventSource.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/DataMovementEventSource.cs @@ -16,6 +16,7 @@ internal class DataMovementEventSource : AzureEventSource private const int EnumerationCompleteEvent = 4; private const int ResumeTransferEvent = 5; private const int ResumeEnumerationCompleteEvent = 6; + private const int UnexpectedTransferFailedEvent = 7; private DataMovementEventSource() : base(EventSourceName) { } @@ -92,5 +93,11 @@ public void ResumeEnumerationComplete(string transferId, int jobPartCount) { WriteEvent(ResumeEnumerationCompleteEvent, transferId, jobPartCount); } + + [Event(UnexpectedTransferFailedEvent, Level = EventLevel.Error, Message = "Transfer [{0}] Transfer failed: {1}")] + public void UnexpectedTransferFailed(string transferId, string errorMessage) + { + WriteEvent(UnexpectedTransferFailedEvent, transferId, errorMessage); + } } } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/DownloadChunkHandler.cs b/sdk/storage/Azure.Storage.DataMovement/src/DownloadChunkHandler.cs index 06c8e225f008..21445576e790 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/DownloadChunkHandler.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/DownloadChunkHandler.cs @@ -136,7 +136,21 @@ await _copyToDestinationFile( if (_isChunkHandlerRunning) { // This will trigger the job part to call Dispose on this object - _ = Task.Run(() => _invokeFailedEventHandler(ex)); + _ = Task.Run(async () => + { + try + { + await _invokeFailedEventHandler(ex).ConfigureAwait(false); + } + catch + { + // Log and swallow any exceptions to prevent crashing the process + DataMovementEventSource.Singleton + .UnexpectedTransferFailed( + nameof(CommitChunkHandler), + ex.ToString()); + } + }); } } } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/TransferInternalState.cs b/sdk/storage/Azure.Storage.DataMovement/src/TransferInternalState.cs index dcbfed107715..c2d898b32faa 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/TransferInternalState.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/TransferInternalState.cs @@ -107,6 +107,9 @@ public bool SetTransferState(TransferState state) // Tell the transfer manager to clean up the completed/paused job. TransferManager.TryRemoveTransfer(_id); + // Remove Transfer Manager reference after no longer needed. + TransferManager = null; + // Once we reach a Completed/Paused, Dispose the CancellationTokenSource to release resources (since it is no longer needed). DisposeCancellationTokenSource(); } From 43782ef649e1fd1417bd201130d62dc22b03507d Mon Sep 17 00:00:00 2001 From: Amanda Nguyen Date: Mon, 29 Sep 2025 10:59:52 -0700 Subject: [PATCH 2/8] Update changelog; change transfermanager use in copy tests to dispose --- .../CHANGELOG.md | 1 + .../CHANGELOG.md | 1 + .../tests/Shared/StartTransferCopyTestBase.cs | 34 ++++++++++++------- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/CHANGELOG.md b/sdk/storage/Azure.Storage.DataMovement.Blobs/CHANGELOG.md index bd45cc443930..b44b2c8d29f8 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/CHANGELOG.md +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/CHANGELOG.md @@ -11,6 +11,7 @@ - `TransferOperation` disposes the `CancellationTokenSource` after transfer reaches a `Completed` or `Paused` state - `TransferManager` uses a `CancellationTokenSource` also does not link the`CancellationToken` passed to it's methods - Removed usage of `CancellationTokenSource` from handling the chunking of large transfers. This only affects transfers that cannot be completed in one request. + - Disposed internal use of `TransferManager` in BlobContainerClient extension methods, `UploadDirectoryAsync(..)` and `DownloadToDirectoryAsync(..)` - Fixed bug where cached referenced `TransferOperation`s from the `TransferManager` were not being cleared on dispose. - Fixed bug where referenced `TransferOperation` from the transfers stored in the `TransferManager` after they reach a `Completed` or `Paused` state where not being removed. diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/CHANGELOG.md b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/CHANGELOG.md index b3f8522d50b3..c2abbce045bb 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/CHANGELOG.md +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/CHANGELOG.md @@ -11,6 +11,7 @@ - `TransferOperation` disposes the `CancellationTokenSource` after transfer reaches a `Completed` or `Paused` state - `TransferManager` uses a `CancellationTokenSource` also does not link the`CancellationToken` passed to it's methods - Removed usage of `CancellationTokenSource` from handling the chunking of large transfers. This only affects transfers that cannot be completed in one request. + - Disposed internal use of `TransferManager` in ShareDirectoryClient extension methods, `UploadDirectoryAsync(..)` and `DownloadToDirectoryAsync(..)` - Fixed bug where cached referenced `TransferOperation`s from the `TransferManager` were not being cleared on dispose. - Fixed bug where referenced `TransferOperation` from the transfers stored in the `TransferManager` after they reach a `Completed` or `Paused` state where not being removed. diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferCopyTestBase.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferCopyTestBase.cs index 1fe2957458cf..64ca6b02d80a 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferCopyTestBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferCopyTestBase.cs @@ -678,10 +678,24 @@ await TestTransferWithTimeout.WaitForCompletionAsync( } #endregion + /// + /// For usage, please dispose of the TransferManager after use. + /// + /// Concurrency to set in TransferManagerOptions. + /// Valid TransferManager that needs to be disposed after use. + private TransferManager CreateTransferManager(int concurrency = default) + { + TransferManagerOptions managerOptions = new TransferManagerOptions() + { + MaximumConcurrency = concurrency, + }; + return new TransferManager(managerOptions); + } + private async Task CreateStartTransfer( TSourceContainerClient sourceContainer, TDestinationContainerClient destinationContainer, - int concurrency, + TransferManager transferManager, bool createFailedCondition = false, TransferOptions options = default, int size = DataMovementTestConstants.KB) @@ -705,13 +719,6 @@ private async Task CreateStartTransfer( StorageResourceItem sourceResource = GetSourceStorageResourceItem(sourceClient); StorageResourceItem destinationResource = GetDestinationStorageResourceItem(destinationClient); - // Create Transfer Manager with single threaded operation - TransferManagerOptions managerOptions = new TransferManagerOptions() - { - MaximumConcurrency = concurrency, - }; - TransferManager transferManager = new TransferManager(managerOptions); - // Start transfer and await for completion. return await transferManager.StartTransferAsync( sourceResource, @@ -725,6 +732,7 @@ public async Task StartTransfer_AwaitCompletion() // Arrange await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); + await using TransferManager transferManager = CreateTransferManager(concurrency: 1); TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); @@ -733,7 +741,7 @@ public async Task StartTransfer_AwaitCompletion() TransferOperation transfer = await CreateStartTransfer( source.Container, destination.Container, - concurrency: 1, + transferManager, options: options); // Act @@ -756,6 +764,7 @@ public async Task StartTransfer_AwaitCompletion_Failed() // Arrange await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); + await using TransferManager transferManager = CreateTransferManager(concurrency: 1); TransferOptions options = new TransferOptions() { @@ -767,7 +776,7 @@ public async Task StartTransfer_AwaitCompletion_Failed() TransferOperation transfer = await CreateStartTransfer( source.Container, destination.Container, - concurrency: 1, + transferManager, createFailedCondition: true, options: options); @@ -801,6 +810,7 @@ public async Task StartTransfer_AwaitCompletion_Skipped() // Arrange await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); + await using TransferManager transferManager = CreateTransferManager(concurrency: 1); // Create transfer options with Skipping available TransferOptions options = new TransferOptions() @@ -813,7 +823,7 @@ public async Task StartTransfer_AwaitCompletion_Skipped() TransferOperation transfer = await CreateStartTransfer( source.Container, destination.Container, - concurrency: 1, + transferManager, createFailedCondition: true, options: options); @@ -860,7 +870,7 @@ private async Task CopyRemoteObjects_VerifyProperties( MaximumTransferChunkSize = chunkSize, }; TestEventsRaised testEventsRaised = new TestEventsRaised(options); - TransferManager transferManager = new TransferManager(); + await using TransferManager transferManager = new TransferManager(); // Act - Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( From 8a5990fd724e5d29e2b1c2d874c33b3a9d860dac Mon Sep 17 00:00:00 2001 From: Amanda Nguyen Date: Thu, 2 Oct 2025 10:43:15 -0700 Subject: [PATCH 3/8] WIP - updating tests to dipose Transfermanager --- .../Infrastructure/DirectoryTransferTest.cs | 6 + .../samples/Sample01b_Migration.cs | 18 +-- .../tests/GetTransfersTests.cs | 16 +-- .../Shared/PauseResumeTransferTestBase.cs | 28 ++-- .../tests/Shared/StartTransferCopyTestBase.cs | 6 +- .../StartTransferDirectoryCopyTestBase.cs | 51 ++++--- .../StartTransferDirectoryDownloadTestBase.cs | 47 +++++-- .../Shared/StartTransferDownloadTestBase.cs | 47 ++++--- .../StartTransferUploadDirectoryTestBase.cs | 127 +++++++++++------- .../Shared/StartTransferUploadTestBase.cs | 40 ++++-- .../tests/TransferOperationTests.cs | 3 +- 11 files changed, 244 insertions(+), 145 deletions(-) diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Azure.Storage.DataMovement.Blobs.Perf/Infrastructure/DirectoryTransferTest.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Azure.Storage.DataMovement.Blobs.Perf/Infrastructure/DirectoryTransferTest.cs index e4171485e7a8..61d95c751c25 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Azure.Storage.DataMovement.Blobs.Perf/Infrastructure/DirectoryTransferTest.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Azure.Storage.DataMovement.Blobs.Perf/Infrastructure/DirectoryTransferTest.cs @@ -34,6 +34,12 @@ public DirectoryTransferTest(TOptions options) : base(options) _transferManager = new TransferManager(managerOptions); } + public override async Task CleanupAsync() + { + await ((IAsyncDisposable)_transferManager).DisposeAsync(); + await base.CleanupAsync(); + } + protected string CreateLocalDirectory(bool populate = false) { string directory = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); diff --git a/sdk/storage/Azure.Storage.DataMovement/samples/Sample01b_Migration.cs b/sdk/storage/Azure.Storage.DataMovement/samples/Sample01b_Migration.cs index 3b11393a2b33..f95d61599ee0 100644 --- a/sdk/storage/Azure.Storage.DataMovement/samples/Sample01b_Migration.cs +++ b/sdk/storage/Azure.Storage.DataMovement/samples/Sample01b_Migration.cs @@ -53,7 +53,7 @@ public async Task UploadBlob() string filePath; Uri blobUri; BlobsStorageResourceProvider blobs; - TransferManager transferManager; + await using TransferManager transferManager = new TransferManager(); #endregion // Create a temporary Lorem Ipsum file on disk to upload @@ -79,7 +79,6 @@ public async Task UploadBlob() try { - transferManager = new TransferManager(); blobs = new BlobsStorageResourceProvider(credential); #region Snippet:DataMovementMigration_UploadSingleFile @@ -104,7 +103,7 @@ public async Task UploadBlobDirectory() string directoryPath, blobDirectoryPath; Uri containerUri; BlobsStorageResourceProvider blobs; - TransferManager transferManager; + await using TransferManager transferManager = new TransferManager(); #endregion // Create a temporary populated directory on disk to upload @@ -129,7 +128,6 @@ public async Task UploadBlobDirectory() try { - transferManager = new TransferManager(); blobs = new BlobsStorageResourceProvider(credential); #region Snippet:DataMovementMigration_UploadBlobDirectory @@ -157,7 +155,7 @@ public async Task DownloadBlob() string filePath; Uri blobUri; BlobsStorageResourceProvider blobs; - TransferManager transferManager; + await using TransferManager transferManager = new TransferManager(); #endregion // Create a temporary populated directory on disk to upload @@ -186,7 +184,6 @@ public async Task DownloadBlob() await container.GetBlobClient(blobName) .UploadAsync(BinaryData.FromString(SampleFileContent)); - transferManager = new TransferManager(); blobs = new BlobsStorageResourceProvider(credential); #region Snippet:DataMovementMigration_DownloadBlob @@ -211,7 +208,7 @@ public async Task DownloadBlobDirectory() string directoryPath, blobDirectoryPath; Uri containerUri; BlobsStorageResourceProvider blobs; - TransferManager transferManager; + await using TransferManager transferManager = new TransferManager(); #endregion // Create a temporary populated directory on disk to upload @@ -236,7 +233,6 @@ public async Task DownloadBlobDirectory() try { - transferManager = new TransferManager(); blobs = new BlobsStorageResourceProvider(credential); #region Snippet:DataMovementMigration_DownloadBlobDirectory @@ -263,7 +259,7 @@ public async Task CopyBlob() // these values provided by your code Uri srcBlobUri, dstBlobUri; BlobsStorageResourceProvider blobs; - TransferManager transferManager; + await using TransferManager transferManager = new TransferManager(); #endregion // Get account and shared key access @@ -295,7 +291,6 @@ public async Task CopyBlob() await container.GetBlobClient(srcBlobName) .UploadAsync(BinaryData.FromString(SampleFileContent)); - transferManager = new TransferManager(); blobs = new BlobsStorageResourceProvider(credential); #region Snippet:DataMovementMigration_CopyBlobToBlob @@ -320,7 +315,7 @@ public async Task CopyBlobToShare() Uri blobUri, fileUri; BlobsStorageResourceProvider blobs; ShareFilesStorageResourceProvider files; - TransferManager transferManager; + await using TransferManager transferManager = new TransferManager(); #endregion // Get account and shared key access @@ -357,7 +352,6 @@ await container.GetBlobClient(blobName) share = new ShareClient(accountUri, credential); await share.CreateIfNotExistsAsync(); - transferManager = new TransferManager(); blobs = new BlobsStorageResourceProvider(credential); files = new ShareFilesStorageResourceProvider(credential); diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/GetTransfersTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/GetTransfersTests.cs index 9522472af77f..5f8ee21d41d5 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/GetTransfersTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/GetTransfersTests.cs @@ -58,7 +58,7 @@ public async Task GetTransfers_Empty() { using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); // Arrange - TransferManager manager = new TransferManager(GetDefaultManagerOptions(testDirectory.DirectoryPath)); + await using TransferManager manager = new TransferManager(GetDefaultManagerOptions(testDirectory.DirectoryPath)); // Act IList transfers = await manager.GetTransfersAsync().ToListAsync(); @@ -80,7 +80,7 @@ public async Task GetTransfers_Populated() }; TransferManagerFactory factory = new TransferManagerFactory(GetDefaultManagerOptions(testDirectory.DirectoryPath)); - TransferManager manager = factory.BuildTransferManager(storedTransfers); + await using TransferManager manager = factory.BuildTransferManager(storedTransfers); // Act IList result = await manager.GetTransfersAsync().ToListAsync(); @@ -120,7 +120,7 @@ public async Task GetTransfers_Filtered( }; TransferManagerFactory factory = new TransferManagerFactory(GetDefaultManagerOptions(testDirectory.DirectoryPath)); - TransferManager manager = factory.BuildTransferManager(storedTransfers); + await using TransferManager manager = factory.BuildTransferManager(storedTransfers); // Act TransferStatus[] status = { new TransferStatus(state, hasFailedItems, hasSkippedItems) }; @@ -154,7 +154,7 @@ public async Task GetTransfers_FilterMultipleStatuses() }; TransferManagerFactory factory = new TransferManagerFactory(GetDefaultManagerOptions(testDirectory.DirectoryPath)); - TransferManager manager = factory.BuildTransferManager(storedTransfers); + await using TransferManager manager = factory.BuildTransferManager(storedTransfers); // Act TransferStatus[] statuses = new TransferStatus[] { @@ -190,7 +190,7 @@ public async Task GetTransfers_Filtered_Empty() }; TransferManagerFactory factory = new TransferManagerFactory(GetDefaultManagerOptions(testDirectory.DirectoryPath)); - TransferManager manager = factory.BuildTransferManager(storedTransfers); + await using TransferManager manager = factory.BuildTransferManager(storedTransfers); // Act - With a transfer status not in the above stored transfers TransferStatus[] statuses = new TransferStatus[] { new TransferStatus(TransferState.Stopping, true, false) }; @@ -218,7 +218,7 @@ public async Task GetTransfers_LocalCheckpointer() { CheckpointStoreOptions = TransferCheckpointStoreOptions.CreateLocalStore(test.DirectoryPath) }; - TransferManager manager = new TransferManager(options); + await using TransferManager manager = new TransferManager(options); // Act IList result = await manager.GetTransfersAsync().ToListAsync(); @@ -260,7 +260,7 @@ public async Task GetResumableTransfers_LocalCheckpointer() { CheckpointStoreOptions = TransferCheckpointStoreOptions.CreateLocalStore(test.DirectoryPath) }; - TransferManager manager = new TransferManager(options); + await using TransferManager manager = new TransferManager(options); // Act IList result = await manager.GetResumableTransfersAsync().ToListAsync(); @@ -299,7 +299,7 @@ public async Task GetResumableTransfers_IgnoresCompleted() { CheckpointStoreOptions = TransferCheckpointStoreOptions.CreateLocalStore(test.DirectoryPath) }; - TransferManager manager = new TransferManager(options); + await using TransferManager manager = new TransferManager(options); // Act IList result = await manager.GetResumableTransfersAsync().ToListAsync(); diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/PauseResumeTransferTestBase.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/PauseResumeTransferTestBase.cs index d102f5e96baf..87903793052e 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/PauseResumeTransferTestBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/PauseResumeTransferTestBase.cs @@ -406,7 +406,7 @@ public async Task TryPauseTransferAsync_Id(TransferDirection transferType) ErrorMode = TransferErrorMode.ContinueOnFailure, ProvidersForResuming = new List() { provider }, }; - TransferManager transferManager = new TransferManager(options); + await using TransferManager transferManager = new TransferManager(options); TestProgressHandler progressHandler = new(); TransferOptions transferOptions = new TransferOptions { @@ -482,7 +482,7 @@ public async Task TryPauseTransferAsync_TransferOperation(TransferDirection tran } }; TestEventsRaised testEventsRaised = new TestEventsRaised(transferOptions); - TransferManager transferManager = new TransferManager(options); + await using TransferManager transferManager = new TransferManager(options); // Add long-running job to pause, if the job is not big enough // then the job might finish before we can pause it. @@ -520,7 +520,7 @@ public async Task TryPauseTransferAsync_TransferOperation(TransferDirection tran } [RecordedTest] - public void TryPauseTransferAsync_Error() + public async Task TryPauseTransferAsync_Error() { // Arrange using DisposingLocalDirectory checkpointerDirectory = DisposingLocalDirectory.GetTestDirectory(); @@ -529,7 +529,7 @@ public void TryPauseTransferAsync_Error() CheckpointStoreOptions = TransferCheckpointStoreOptions.CreateLocalStore(checkpointerDirectory.DirectoryPath), ErrorMode = TransferErrorMode.ContinueOnFailure }; - TransferManager transferManager = new TransferManager(options); + await using TransferManager transferManager = new TransferManager(options); // Act / Assert Assert.CatchAsync(async () => await transferManager.PauseTransferAsync("bad transfer Id")); @@ -566,7 +566,7 @@ public async Task TryPauseTransferAsync_AlreadyPaused(TransferDirection transfer } }; TestEventsRaised testEventsRaised = new TestEventsRaised(transferOptions); - TransferManager transferManager = new TransferManager(options); + await using TransferManager transferManager = new TransferManager(options); // Add long-running job to pause, if the job is not big enough // then the job might finish before we can pause it. @@ -632,7 +632,7 @@ public async Task PauseThenResumeTransferAsync(TransferDirection transferType) }; TransferOptions transferOptions = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(transferOptions); - TransferManager transferManager = new TransferManager(options); + await using TransferManager transferManager = new TransferManager(options); long size = DataMovementTestConstants.KB * 100; (StorageResource sResource, StorageResource dResource) = await CreateStorageResourcesAsync( @@ -720,7 +720,7 @@ public async Task ResumeTransferAsync(TransferDirection transferType) }; TransferOptions transferOptions = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(transferOptions); - TransferManager transferManager = new TransferManager(options); + await using TransferManager transferManager = new TransferManager(options); long size = DataMovementTestConstants.KB * 100; (StorageResource sResource, StorageResource dResource) = await CreateStorageResourcesAsync( @@ -784,7 +784,7 @@ public async Task ResumeTransferAsync_Options(TransferDirection transferType) ErrorMode = TransferErrorMode.ContinueOnFailure, ProvidersForResuming = new List() { provider }, }; - TransferManager transferManager = new TransferManager(options); + await using TransferManager transferManager = new TransferManager(options); Metadata metadata = DataProvider.BuildMetadata(); string contentLanguage = "en-US"; @@ -849,7 +849,7 @@ public async Task TryPauseTransferAsync_Id_Directory(TransferDirection transferT ErrorMode = TransferErrorMode.ContinueOnFailure, ProvidersForResuming = new List() { provider }, }; - TransferManager transferManager = new TransferManager(options); + await using TransferManager transferManager = new TransferManager(options); TransferOptions transferOptions = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(transferOptions); @@ -898,7 +898,7 @@ public async Task TryPauseTransferAsync_TransferOperation_Directory(TransferDire ErrorMode = TransferErrorMode.ContinueOnFailure, ProvidersForResuming = new List() { provider }, }; - TransferManager transferManager = new TransferManager(options); + await using TransferManager transferManager = new TransferManager(options); TransferOptions transferOptions = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(transferOptions); @@ -947,7 +947,7 @@ public async Task TryPauseTransferAsync_AlreadyPaused_Directory(TransferDirectio ErrorMode = TransferErrorMode.ContinueOnFailure, ProvidersForResuming = new List() { provider }, }; - TransferManager transferManager = new TransferManager(options); + await using TransferManager transferManager = new TransferManager(options); TransferOptions transferOptions = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(transferOptions); @@ -1007,7 +1007,7 @@ public async Task PauseThenResumeTransferAsync_Directory(TransferDirection trans ErrorMode = TransferErrorMode.ContinueOnFailure, ProvidersForResuming = new List() { provider }, }; - TransferManager transferManager = new TransferManager(options); + await using TransferManager transferManager = new TransferManager(options); TransferOptions transferOptions = new TransferOptions() { InitialTransferSize = DataMovementTestConstants.KB, @@ -1091,7 +1091,7 @@ public async Task ResumeTransferAsync_Directory(TransferDirection transferType) ErrorMode = TransferErrorMode.ContinueOnFailure, ProvidersForResuming = new List() { provider }, }; - TransferManager transferManager = new TransferManager(options); + await using TransferManager transferManager = new TransferManager(options); TransferOptions transferOptions = new TransferOptions() { InitialTransferSize = DataMovementTestConstants.KB, @@ -1182,7 +1182,7 @@ public async Task ResumeTransferAsync_Directory_Large( ErrorMode = TransferErrorMode.ContinueOnFailure, ProvidersForResuming = new List() { provider }, }; - TransferManager transferManager = new TransferManager(options); + await using TransferManager transferManager = new TransferManager(options); long size = DataMovementTestConstants.MB; (StorageResource sResource, StorageResource dResource) = await CreateStorageResourceContainersAsync( diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferCopyTestBase.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferCopyTestBase.cs index 0873fcb2e8f8..00aa23bf6967 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferCopyTestBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferCopyTestBase.cs @@ -296,7 +296,7 @@ private async Task CopyRemoteObjectsAndVerify( List copyObjectInfo = new List(objectCount); // Initialize transfer manager - TransferManager transferManager = new TransferManager(transferManagerOptions); + await using TransferManager transferManager = new TransferManager(transferManagerOptions); // Upload set of VerifyCopyFromUriInfo Remote Objects to Copy for (int i = 0; i < objectCount; i++) @@ -582,7 +582,7 @@ public async Task SourceObjectToDestinationObject_Skip_Exists() StorageResourceItem destinationResource = GetDestinationStorageResourceItem(destinationClient); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - TransferManager transferManager = new TransferManager(); + await using TransferManager transferManager = new TransferManager(); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -642,7 +642,7 @@ public async Task SourceObjectToDestinationObject_Failure_Exists() objectLength: size); StorageResourceItem sourceResource = GetSourceStorageResourceItem(sourceClient); StorageResourceItem destinationResource = GetDestinationStorageResourceItem(destinationClient); - TransferManager transferManager = new TransferManager(); + await using TransferManager transferManager = new TransferManager(); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs index ff34fabc93b9..c54284d3bbac 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs @@ -255,7 +255,7 @@ private async Task CopyDirectoryAndVerifyAsync( }; // Initialize transferManager - TransferManager transferManager = new TransferManager(transferManagerOptions); + await using TransferManager transferManager = new TransferManager(transferManagerOptions); StorageResourceContainer sourceResource = GetSourceStorageResourceContainer(sourceContainer, sourcePrefix); @@ -383,7 +383,7 @@ public async Task DirectoryToDirectory_EmptyFolder() ErrorMode = TransferErrorMode.ContinueOnFailure, MaximumConcurrency = 1, }; - TransferManager transferManager = new TransferManager(managerOptions); + await using TransferManager transferManager = new TransferManager(managerOptions); TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); @@ -694,7 +694,7 @@ internal async Task CreateDirectoryTreeAsync( private async Task CreateStartTransfer( TSourceContainerClient sourceContainer, TDestinationContainerClient destinationContainer, - int concurrency, + TransferManager transferManager, bool createFailedCondition = false, TransferOptions options = default, int size = DataMovementTestConstants.KB) @@ -718,20 +718,27 @@ private async Task CreateStartTransfer( await CreateObjectInDestinationAsync(destinationContainer, size, fullDestPath); } - // Create Transfer Manager with single threaded operation - TransferManagerOptions managerOptions = new TransferManagerOptions() - { - MaximumConcurrency = concurrency, - }; - TransferManager transferManager = new TransferManager(managerOptions); - - // Start transfer and await for completion. + // Start transfer. return await transferManager.StartTransferAsync( sourceResource, destinationResource, options).ConfigureAwait(false); } + /// + /// For usage, please dispose of the TransferManager after use. + /// + /// Concurrency to set in TransferManagerOptions. + /// Valid TransferManager that needs to be disposed after use. + private TransferManager CreateTransferManager(int concurrency = default) + { + TransferManagerOptions managerOptions = new TransferManagerOptions() + { + MaximumConcurrency = concurrency, + }; + return new TransferManager(managerOptions); + } + [RecordedTest] public async Task StartTransfer_AwaitCompletion() { @@ -739,13 +746,16 @@ public async Task StartTransfer_AwaitCompletion() await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); + // Create Transfer Manager with single threaded operation + await using TransferManager transferManager = CreateTransferManager(concurrency:1); + // Create transfer to do a AwaitCompletion TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); TransferOperation transfer = await CreateStartTransfer( source.Container, destination.Container, - 1, + transferManager, options: options); // Act @@ -770,6 +780,9 @@ public async Task StartTransfer_AwaitCompletion_Failed() await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); + // Create Transfer Manager with single threaded operation + await using TransferManager transferManager = CreateTransferManager(concurrency: 1); + TransferOptions options = new TransferOptions() { CreationMode = StorageResourceCreationMode.FailIfExists @@ -780,7 +793,7 @@ public async Task StartTransfer_AwaitCompletion_Failed() TransferOperation transfer = await CreateStartTransfer( source.Container, destination.Container, - 1, + transferManager, createFailedCondition: true, options: options); @@ -807,6 +820,9 @@ public async Task StartTransfer_AwaitCompletion_Skipped() await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); + // Create Transfer Manager with single threaded operation + await using TransferManager transferManager = CreateTransferManager(concurrency: 1); + // Create transfer options with Skipping available TransferOptions options = new TransferOptions() { @@ -818,7 +834,7 @@ public async Task StartTransfer_AwaitCompletion_Skipped() TransferOperation transfer = await CreateStartTransfer( source.Container, destination.Container, - 1, + transferManager, createFailedCondition: true, options: options); @@ -845,6 +861,9 @@ public async Task StartTransfer_AwaitCompletion_Failed_SmallChunks() await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); + // Create Transfer Manager with single threaded operation + await using TransferManager transferManager = CreateTransferManager(concurrency: 1); + TransferOptions options = new TransferOptions() { CreationMode = StorageResourceCreationMode.FailIfExists, @@ -857,7 +876,7 @@ public async Task StartTransfer_AwaitCompletion_Failed_SmallChunks() TransferOperation transfer = await CreateStartTransfer( source.Container, destination.Container, - 1, + transferManager, createFailedCondition: true, options: options, size: DataMovementTestConstants.KB * 4); @@ -916,7 +935,7 @@ private async Task CopyRemoteObjects_VerifyProperties( // Create Transfer Manager TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - TransferManager transferManager = new TransferManager(); + await using TransferManager transferManager = new TransferManager(); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryDownloadTestBase.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryDownloadTestBase.cs index efca0d8b5797..501772a9413b 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryDownloadTestBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryDownloadTestBase.cs @@ -242,7 +242,7 @@ public async Task DownloadDirectoryAsync_Empty() await SetupSourceDirectoryAsync(test.Container, sourceDirectoryName, new(), cancellationTokenSource.Token); // Initialize transferManager - TransferManager transferManager = new TransferManager(); + await using TransferManager transferManager = new TransferManager(); TransferOptions options = new TransferOptions(); TestEventsRaised testEventRaised = new TestEventsRaised(options); @@ -459,7 +459,7 @@ await SetupSourceDirectoryAsync( private async Task CreateStartTransfer( TContainerClient containerClient, string destinationFolder, - int concurrency, + TransferManager transferManager, TransferOptions options = default, int size = Constants.KB, CancellationToken cancellationToken = default) @@ -472,14 +472,6 @@ private async Task CreateStartTransfer( StorageResourceContainer sourceResource = GetStorageResourceContainer(containerClient, sourcePrefix); StorageResource destinationResource = LocalFilesStorageResourceProvider.FromDirectory(destinationFolder); - // Create Transfer Manager with single threaded operation - TransferManagerOptions managerOptions = new TransferManagerOptions() - { - MaximumConcurrency = concurrency, - ErrorMode = TransferErrorMode.StopOnAnyFailure - }; - TransferManager transferManager = new TransferManager(managerOptions); - // Start transfer and await for completion. return await transferManager.StartTransferAsync( sourceResource, @@ -487,6 +479,21 @@ private async Task CreateStartTransfer( options).ConfigureAwait(false); } + /// + /// For usage, please dispose of the TransferManager after use. + /// + /// Concurrency to set in TransferManagerOptions. + /// Valid TransferManager that needs to be disposed after use. + private TransferManager CreateTransferManager(int concurrency = default) + { + TransferManagerOptions managerOptions = new TransferManagerOptions() + { + MaximumConcurrency = concurrency, + ErrorMode = TransferErrorMode.StopOnAnyFailure + }; + return new TransferManager(managerOptions); + } + [Test] public async Task StartTransfer_AwaitCompletion() { @@ -496,13 +503,16 @@ public async Task StartTransfer_AwaitCompletion() string destinationFolder = CreateRandomDirectory(testDirectory.DirectoryPath); using CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + // Create Transfer Manager with single threaded operation + await using TransferManager transferManager = CreateTransferManager(concurrency: 1); + // Create transfer to do a AwaitCompletion TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); TransferOperation transfer = await CreateStartTransfer( test.Container, destinationFolder, - 1, + transferManager, options: options, cancellationToken: cancellationTokenSource.Token); @@ -529,6 +539,9 @@ public async Task StartTransfer_AwaitCompletion_Failed() string destinationFolder = CreateRandomDirectory(testDirectory.DirectoryPath); using CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + // Create Transfer Manager with single threaded operation + await using TransferManager transferManager = CreateTransferManager(concurrency: 1); + TransferOptions options = new TransferOptions() { CreationMode = StorageResourceCreationMode.FailIfExists @@ -542,7 +555,7 @@ public async Task StartTransfer_AwaitCompletion_Failed() TransferOperation transfer = await CreateStartTransfer( test.Container, destinationFolder, - 1, + transferManager, options: options, cancellationToken: cancellationTokenSource.Token); @@ -570,6 +583,9 @@ public async Task StartTransfer_AwaitCompletion_Skipped() string destinationFolder = CreateRandomDirectory(testDirectory.DirectoryPath); using CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + // Create Transfer Manager with single threaded operation + await using TransferManager transferManager = CreateTransferManager(concurrency: 1); + // Create transfer options with Skipping available TransferOptions options = new TransferOptions() { @@ -584,7 +600,7 @@ public async Task StartTransfer_AwaitCompletion_Skipped() TransferOperation transfer = await CreateStartTransfer( test.Container, destinationFolder, - 1, + transferManager, options: options, cancellationToken: cancellationTokenSource.Token); @@ -612,6 +628,9 @@ public async Task StartTransfer_AwaitCompletion_Failed_SmallChunks() string destinationFolder = CreateRandomDirectory(testDirectory.DirectoryPath); using CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + // Create Transfer Manager with single threaded operation + await using TransferManager transferManager = CreateTransferManager(concurrency: 1); + TransferOptions options = new TransferOptions() { CreationMode = StorageResourceCreationMode.FailIfExists, @@ -627,7 +646,7 @@ public async Task StartTransfer_AwaitCompletion_Failed_SmallChunks() TransferOperation transfer = await CreateStartTransfer( test.Container, destinationFolder, - 1, + transferManager, options: options, size: Constants.KB * 4, cancellationToken: cancellationTokenSource.Token); diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDownloadTestBase.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDownloadTestBase.cs index 593ab5cfcfce..41f6a6b57654 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDownloadTestBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDownloadTestBase.cs @@ -114,7 +114,7 @@ protected string GetNewObjectName() private async Task CreateStartTransfer( TContainerClient containerClient, string localDirectoryPath, - int concurrency, + TransferManager transferManager, bool createFailedCondition = false, TransferOptions options = default, int size = DataMovementTestConstants.KB) @@ -141,13 +141,6 @@ private async Task CreateStartTransfer( StorageResourceItem sourceResource = GetStorageResourceItem(TObjectClient); StorageResource destinationResource = LocalFilesStorageResourceProvider.FromFile(destFile); - // Create Transfer Manager with single threaded operation - TransferManagerOptions managerOptions = new TransferManagerOptions() - { - MaximumConcurrency = concurrency, - }; - TransferManager transferManager = new TransferManager(managerOptions); - // Start transfer and await for completion. return await transferManager.StartTransferAsync( sourceResource, @@ -155,6 +148,21 @@ private async Task CreateStartTransfer( options).ConfigureAwait(false); } + /// + /// For usage, please dispose of the TransferManager after use. + /// + /// Concurrency to set in TransferManagerOptions. + /// Valid TransferManager that needs to be disposed after use. + private TransferManager CreateTransferManager(int concurrency = default) + { + TransferManagerOptions managerOptions = new TransferManagerOptions() + { + MaximumConcurrency = concurrency, + ErrorMode = TransferErrorMode.StopOnAnyFailure + }; + return new TransferManager(managerOptions); + } + [RecordedTest] public async Task StartTransfer_AwaitCompletion() { @@ -162,13 +170,16 @@ public async Task StartTransfer_AwaitCompletion() await using IDisposingContainer test = await GetDisposingContainerAsync(); using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); + // Create Transfer Manager with single threaded operation + await using TransferManager transferManager = CreateTransferManager(concurrency: 1); + // Create transfer to do a AwaitCompletion TransferOptions options = new TransferOptions(); TestEventsRaised failureTransferHolder = new TestEventsRaised(options); TransferOperation transfer = await CreateStartTransfer( containerClient: test.Container, localDirectoryPath: testDirectory.DirectoryPath, - concurrency: 1, + transferManager: transferManager, options: options); // Act @@ -192,6 +203,9 @@ public async Task StartTransfer_AwaitCompletion_Failed() await using IDisposingContainer test = await GetDisposingContainerAsync(); using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); + // Create Transfer Manager with single threaded operation + await using TransferManager transferManager = CreateTransferManager(concurrency: 1); + TransferOptions options = new TransferOptions() { CreationMode = StorageResourceCreationMode.FailIfExists @@ -202,7 +216,7 @@ public async Task StartTransfer_AwaitCompletion_Failed() TransferOperation transfer = await CreateStartTransfer( containerClient: test.Container, localDirectoryPath: testDirectory.DirectoryPath, - concurrency: 1, + transferManager: transferManager, createFailedCondition: true, options: options); @@ -230,6 +244,9 @@ public async Task StartTransfer_AwaitCompletion_Skipped() await using IDisposingContainer test = await GetDisposingContainerAsync(); using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); + // Create Transfer Manager with single threaded operation + await using TransferManager transferManager = CreateTransferManager(concurrency: 1); + // Create transfer options with Skipping available TransferOptions options = new TransferOptions() { @@ -241,7 +258,7 @@ public async Task StartTransfer_AwaitCompletion_Skipped() TransferOperation transfer = await CreateStartTransfer( containerClient: test.Container, localDirectoryPath: testDirectory.DirectoryPath, - concurrency: 1, + transferManager: transferManager, createFailedCondition: true, options: options); @@ -286,7 +303,7 @@ public VerifyDownloadObjectContentInfo( /// /// Upload and verify the contents of the object /// - /// By default in this function an event arguement will be added to the options event handler + /// By default in this function an event argument will be added to the options event handler /// to detect when the upload has finished. /// /// @@ -329,7 +346,7 @@ private async Task DownloadObjectsAndVerify( List downloadedObjectInfo = new List(objectCount); // Initialize TransferManager - TransferManager transferManager = new TransferManager(transferManagerOptions); + await using TransferManager transferManager = new TransferManager(transferManagerOptions); // Upload set of VerifyDownloadObjectContentInfo objects to download for (int i = 0; i < objectCount; i++) { @@ -498,7 +515,7 @@ public async Task RemoteObjectToLocal_Skip_Exists() return Task.CompletedTask; }; TestEventsRaised testEventsRaised = new(options); - TransferManager transferManager = new TransferManager(); + await using TransferManager transferManager = new TransferManager(); StorageResource destinationResource = LocalFilesStorageResourceProvider.FromFile(destFile); @@ -551,7 +568,7 @@ public async Task RemoteObjectToLocal_Failure_Exists() StorageResourceItem sourceResource = GetStorageResourceItem(sourceClient); StorageResource destinationResource = LocalFilesStorageResourceProvider.FromFile(destFile); - TransferManager transferManager = new TransferManager(); + await using TransferManager transferManager = new TransferManager(); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferUploadDirectoryTestBase.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferUploadDirectoryTestBase.cs index ddaf7ceeaae9..dd4ef9d3d1ea 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferUploadDirectoryTestBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferUploadDirectoryTestBase.cs @@ -132,7 +132,7 @@ private async Task UploadDirectoryAndVerifyAsync( string sourceLocalDirectoryPath, TContainerClient destinationContainer, int expectedTransfers, - TransferManagerOptions transferManagerOptions = default, + TransferManager transferManager, TransferOptions options = default, CancellationToken cancellationToken = default) { @@ -140,17 +140,12 @@ private async Task UploadDirectoryAndVerifyAsync( options ??= new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - transferManagerOptions ??= new TransferManagerOptions() - { - ErrorMode = TransferErrorMode.ContinueOnFailure - }; - StorageResourceContainer sourceResource = LocalFilesStorageResourceProvider.FromDirectory(sourceLocalDirectoryPath); StorageResourceContainer destinationResource = GetStorageResourceContainer(destinationContainer); await new TransferValidator() { - TransferManager = new(transferManagerOptions) + TransferManager = transferManager }.TransferAndVerifyAsync( sourceResource, destinationResource, @@ -160,6 +155,20 @@ private async Task UploadDirectoryAndVerifyAsync( options, cancellationToken); } + + /// + /// For usage, please dispose of the TransferManager after use. + /// + /// Concurrency to set in TransferManagerOptions. + /// Valid TransferManager that needs to be disposed after use. + private TransferManager CreateTransferManager() + { + TransferManagerOptions transferManagerOptions = new TransferManagerOptions() + { + ErrorMode = TransferErrorMode.ContinueOnFailure + }; + return new TransferManager(transferManagerOptions); + } #endregion [RecordedTest] @@ -175,6 +184,7 @@ public async Task Upload(long objectSize, int waitTimeInSec) // Arrange using DisposingLocalDirectory disposingLocalDirectory = DisposingLocalDirectory.GetTestDirectory(); await using IDisposingContainer test = await GetDisposingContainerAsync(); + await using TransferManager transferManager = CreateTransferManager(); List files = new() { @@ -192,6 +202,7 @@ await UploadDirectoryAndVerifyAsync( disposingLocalDirectory.DirectoryPath, test.Container, expectedTransfers: files.Count, + transferManager: transferManager, cancellationToken: cts.Token); } } @@ -302,33 +313,30 @@ await SetupDirectoryAsync( CreationMode = StorageResourceCreationMode.SkipIfExists }; TestEventsRaised testEventsRaised = new TestEventsRaised(options); - TransferManagerOptions transferManagerOptions = new() + await using TransferManager transferManager = CreateTransferManager(); { - ErrorMode = TransferErrorMode.ContinueOnFailure - }; + StorageResourceContainer sourceResource = LocalFilesStorageResourceProvider.FromDirectory(disposingLocalDirectory.DirectoryPath); + StorageResourceContainer destinationResource = GetStorageResourceContainer(test.Container); + TransferOperation transfer = await transferManager.StartTransferAsync(sourceResource, destinationResource, options, cts.Token); + await TestTransferWithTimeout.WaitForCompletionAsync( + transfer, + testEventsRaised, + cts.Token); - StorageResourceContainer sourceResource = LocalFilesStorageResourceProvider.FromDirectory(disposingLocalDirectory.DirectoryPath); - StorageResourceContainer destinationResource = GetStorageResourceContainer(test.Container); - TransferOperation transfer = await new TransferManager(transferManagerOptions) - .StartTransferAsync(sourceResource, destinationResource, options, cts.Token); - await TestTransferWithTimeout.WaitForCompletionAsync( - transfer, - testEventsRaised, - cts.Token); + // check if expected files exist, but not necessarily for contents + await testEventsRaised.AssertContainerCompletedWithSkippedCheck(preexistingFileCount); - // check if expected files exist, but not necessarily for contents - await testEventsRaised.AssertContainerCompletedWithSkippedCheck(preexistingFileCount); - - // Verify all files exist, meaning files without conflict were transferred. - List localFiles = (await TransferValidator.GetLocalFileLister(disposingLocalDirectory.DirectoryPath) - .Invoke(cts.Token)) - .Select(item => item.RelativePath) - .ToList(); - List destinationObjects = (await GetStorageResourceLister(test.Container) - .Invoke(cts.Token)) - .Select(item => item.RelativePath) - .ToList(); - Assert.That(localFiles, Is.EquivalentTo(destinationObjects)); + // Verify all files exist, meaning files without conflict were transferred. + List localFiles = (await TransferValidator.GetLocalFileLister(disposingLocalDirectory.DirectoryPath) + .Invoke(cts.Token)) + .Select(item => item.RelativePath) + .ToList(); + List destinationObjects = (await GetStorageResourceLister(test.Container) + .Invoke(cts.Token)) + .Select(item => item.RelativePath) + .ToList(); + Assert.That(localFiles, Is.EquivalentTo(destinationObjects)); + } } } @@ -361,12 +369,16 @@ await SetupDirectoryAsync( disposingLocalDirectory.DirectoryPath, files.Select(path => (path, DefaultObjectSize)).ToList(), cts.Token); - await UploadDirectoryAndVerifyAsync( + await using TransferManager transferManager = CreateTransferManager(); + { + await UploadDirectoryAndVerifyAsync( disposingLocalDirectory.DirectoryPath, test.Container, expectedTransfers: files.Count, + transferManager: transferManager, options: options, cancellationToken: cts.Token); + } } } @@ -398,10 +410,12 @@ await SetupDirectoryAsync( disposingLocalDirectory.DirectoryPath, files.Select(path => (path, objectSize)).ToList(), cts.Token); + await using TransferManager transferManager = CreateTransferManager(); await UploadDirectoryAndVerifyAsync( disposingLocalDirectory.DirectoryPath, test.Container, expectedTransfers: files.Count, + transferManager: transferManager, cancellationToken: cts.Token); } } @@ -430,13 +444,15 @@ void BuildFolders(string path, int depth) } BuildFolders(disposingLocalDirectory.DirectoryPath, folderDepth); + await using (TransferManager transferManager = CreateTransferManager()) using (CancellationTokenSource cts = TestHelper.GetTimeoutTokenSource(waitTimeInSec)) { await UploadDirectoryAndVerifyAsync( - disposingLocalDirectory.DirectoryPath, - test.Container, - expectedTransfers: 0, - cancellationToken: cts.Token); + disposingLocalDirectory.DirectoryPath, + test.Container, + expectedTransfers: 0, + transferManager: transferManager, + cancellationToken: cts.Token); } } @@ -471,11 +487,13 @@ await SetupDirectoryAsync( disposingLocalDirectory.DirectoryPath, files.Select(path => (path, DefaultObjectSize)).ToList(), cts.Token); + await using TransferManager transferManager = CreateTransferManager(); await UploadDirectoryAndVerifyAsync( disposingLocalDirectory.DirectoryPath, test.Container, - expectedTransfers: 1 << folderDepth, - cancellationToken: cts.Token); + expectedTransfers: 1 << folderDepth, + transferManager: transferManager, + cancellationToken: cts.Token); } } @@ -500,10 +518,13 @@ await SetupDirectoryAsync( disposingLocalDirectory.DirectoryPath, files.Select(path => (path, DefaultObjectSize)).ToList(), cts.Token); + + await using TransferManager transferManager = CreateTransferManager(); await UploadDirectoryAndVerifyAsync( disposingLocalDirectory.DirectoryPath, test.Container, expectedTransfers: 1, + transferManager: transferManager, cancellationToken: cts.Token); } } @@ -528,6 +549,7 @@ public async Task Upload_SpecialChars(string prefix) string.Join("/", "space folder", "space file"), ]; + await using TransferManager transferManager = CreateTransferManager(); using (CancellationTokenSource cts = TestHelper.GetTimeoutTokenSource(30)) { await SetupDirectoryAsync( @@ -538,6 +560,7 @@ await UploadDirectoryAndVerifyAsync( disposingLocalDirectory.DirectoryPath, test.Container, expectedTransfers: files.Count, + transferManager: transferManager, cancellationToken: cts.Token); } } @@ -551,18 +574,22 @@ public async Task Upload_TrailingSlash() List files = [ "file1", "file2", "dir1/file1" ]; using CancellationTokenSource cancellationTokenSource = TestHelper.GetTimeoutTokenSource(30); - await SetupDirectoryAsync( - disposingLocalDirectory.DirectoryPath, - files.Select(path => (path, (long)Constants.KB)).ToList(), - cancellationTokenSource.Token); - - // Intentionally append trailing slash - string sourcePath = disposingLocalDirectory.DirectoryPath + Path.DirectorySeparatorChar; - await UploadDirectoryAndVerifyAsync( - sourcePath, - test.Container, - expectedTransfers: files.Count, - cancellationToken: cancellationTokenSource.Token); + { + await SetupDirectoryAsync( + disposingLocalDirectory.DirectoryPath, + files.Select(path => (path, (long)Constants.KB)).ToList(), + cancellationTokenSource.Token); + + // Intentionally append trailing slash + string sourcePath = disposingLocalDirectory.DirectoryPath + Path.DirectorySeparatorChar; + await using TransferManager transferManager = CreateTransferManager(); + await UploadDirectoryAndVerifyAsync( + sourcePath, + test.Container, + expectedTransfers: files.Count, + transferManager: transferManager, + cancellationToken: cancellationTokenSource.Token); + } } } } diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferUploadTestBase.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferUploadTestBase.cs index 2439b023af04..dfad36c6d729 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferUploadTestBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferUploadTestBase.cs @@ -120,7 +120,7 @@ protected string GetNewObjectName() private async Task CreateStartTransfer( TContainerClient containerClient, string localDirectoryPath, - int concurrency, + TransferManager transferManager, bool createFailedCondition = false, TransferOptions options = default, int size = DataMovementTestConstants.KB) @@ -147,13 +147,6 @@ private async Task CreateStartTransfer( } StorageResource sourceResource = LocalFilesStorageResourceProvider.FromFile(localSourceFile); - // Create Transfer Manager with single threaded operation - TransferManagerOptions managerOptions = new TransferManagerOptions() - { - MaximumConcurrency = concurrency, - }; - TransferManager transferManager = new TransferManager(managerOptions); - // Start transfer and await for completion. return await transferManager.StartTransferAsync( sourceResource, @@ -161,6 +154,20 @@ private async Task CreateStartTransfer( options).ConfigureAwait(false); } + /// + /// For usage, please dispose of the TransferManager after use. + /// + /// Concurrency to set in TransferManagerOptions. + /// Valid TransferManager that needs to be disposed after use. + private TransferManager CreateTransferManager(int concurrency = default) + { + TransferManagerOptions managerOptions = new TransferManagerOptions() + { + MaximumConcurrency = concurrency, + }; + return new TransferManager(managerOptions); + } + [RecordedTest] public async Task StartTransfer_AwaitCompletion() { @@ -168,6 +175,9 @@ public async Task StartTransfer_AwaitCompletion() using DisposingLocalDirectory disposingLocalDirectory = DisposingLocalDirectory.GetTestDirectory(); await using IDisposingContainer test = await GetDisposingContainerAsync(); + // Create Transfer Manager with single threaded operation + await using TransferManager transferManager = CreateTransferManager(concurrency: 1); + TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); @@ -175,7 +185,7 @@ public async Task StartTransfer_AwaitCompletion() TransferOperation transfer = await CreateStartTransfer( containerClient: test.Container, localDirectoryPath: disposingLocalDirectory.DirectoryPath, - 1, + transferManager: transferManager, options: options); // Act @@ -199,6 +209,9 @@ public async Task StartTransfer_AwaitCompletion_Failed() using DisposingLocalDirectory disposingLocalDirectory = DisposingLocalDirectory.GetTestDirectory(); await using IDisposingContainer test = await GetDisposingContainerAsync(); + // Create Transfer Manager with single threaded operation + await using TransferManager transferManager = CreateTransferManager(concurrency: 1); + TransferOptions options = new TransferOptions() { CreationMode = StorageResourceCreationMode.FailIfExists @@ -209,7 +222,7 @@ public async Task StartTransfer_AwaitCompletion_Failed() TransferOperation transfer = await CreateStartTransfer( containerClient: test.Container, localDirectoryPath: disposingLocalDirectory.DirectoryPath, - concurrency: 1, + transferManager: transferManager, createFailedCondition: true, options: options); @@ -245,6 +258,9 @@ public async Task StartTransfer_AwaitCompletion_Skipped() using DisposingLocalDirectory disposingLocalDirectory = DisposingLocalDirectory.GetTestDirectory(); await using IDisposingContainer test = await GetDisposingContainerAsync(); + // Create Transfer Manager with single threaded operation + await using TransferManager transferManager = CreateTransferManager(concurrency: 1); + // Create transfer options with Skipping available TransferOptions options = new TransferOptions() { @@ -256,7 +272,7 @@ public async Task StartTransfer_AwaitCompletion_Skipped() TransferOperation transfer = await CreateStartTransfer( containerClient: test.Container, localDirectoryPath: disposingLocalDirectory.DirectoryPath, - concurrency: 1, + transferManager: transferManager, createFailedCondition: true, options: options); @@ -343,7 +359,7 @@ private async Task UploadResourceAndVerify( List uploadedObjectInfo = new List(objectCount); // Initialize TransferManager - TransferManager transferManager = new TransferManager(transferManagerOptions); + await using TransferManager transferManager = new TransferManager(transferManagerOptions); // Set up file to upload for (int i = 0; i < objectCount; i++) diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/TransferOperationTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/TransferOperationTests.cs index 3271e998b924..4335f2d30332 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/TransferOperationTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/TransferOperationTests.cs @@ -170,10 +170,11 @@ public async Task TryPauseAsync_CancellationToken() // Arrange string transferId = GetNewTransferId(); + await using TransferManager transferManager = new TransferManager(); TransferOperation transfer = new TransferOperation( id: transferId, status: InProgressStatus); - transfer.TransferManager = new TransferManager(); + transfer.TransferManager = transferManager; CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(1)); try From 2d41f286c3f731c5279c5eb45dab063923ca1243 Mon Sep 17 00:00:00 2001 From: Amanda Nguyen Date: Thu, 2 Oct 2025 14:48:48 -0700 Subject: [PATCH 4/8] Update tests to dispose TransferManager after usage --- .../src/BlobContainerClientExtensions.cs | 40 ++++++++--------- .../tests/PageBlobStartTransferCopyTests.cs | 2 +- .../tests/ProgressHandlerTests.cs | 2 +- .../tests/StartTransferCheckpointerTests.cs | 2 +- .../src/ShareDirectoryClientExtensions.cs | 44 +++++++++---------- .../ShareDirectoryStartTransferCopyTests.cs | 34 +++++++------- ...hareDirectoryStartTransferDownloadTests.cs | 4 +- .../tests/ShareFileStartTransferCopyTests.cs | 32 +++++++------- .../ShareFileStartTransferDownloadTests.cs | 4 +- .../Shared/StartTransferUploadTestBase.cs | 4 +- 10 files changed, 80 insertions(+), 88 deletions(-) diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobContainerClientExtensions.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobContainerClientExtensions.cs index 467b47365159..bcacd2629a37 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobContainerClientExtensions.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobContainerClientExtensions.cs @@ -14,6 +14,8 @@ namespace Azure.Storage.Blobs /// public static class BlobContainerClientExtensions { + private static Lazy s_defaultTransferManager = new Lazy(() => new TransferManager(default)); + /// /// Uploads the entire contents of local directory to the blob container. /// @@ -87,20 +89,18 @@ public static async Task UploadDirectoryAsync( StorageResource localDirectory = LocalFilesStorageResourceProvider.FromDirectory(localDirectoryPath); StorageResource blobDirectory = BlobsStorageResourceProvider.FromClient(client, options?.BlobContainerOptions); - await using (TransferManager transferManager = new TransferManager()) - { - TransferOperation transfer = await transferManager.StartTransferAsync( - localDirectory, - blobDirectory, - options?.TransferOptions, - cancellationToken).ConfigureAwait(false); + TransferOperation transfer = await s_defaultTransferManager.Value.StartTransferAsync( + localDirectory, + blobDirectory, + options?.TransferOptions, + cancellationToken).ConfigureAwait(false); - if (waitUntil == WaitUntil.Completed) - { - await transfer.WaitForCompletionAsync(cancellationToken).ConfigureAwait(false); - } - return transfer; + if (waitUntil == WaitUntil.Completed) + { + await transfer.WaitForCompletionAsync(cancellationToken).ConfigureAwait(false); } + + return transfer; } /// @@ -176,22 +176,18 @@ public static async Task DownloadToDirectoryAsync( StorageResource localDirectory = LocalFilesStorageResourceProvider.FromDirectory(localDirectoryPath); StorageResource blobDirectory = BlobsStorageResourceProvider.FromClient(client, options?.BlobContainerOptions); - // Create a new TransferManager for each operation - await using (TransferManager transferManager = new TransferManager()) - { - TransferOperation transfer = await transferManager.StartTransferAsync( + TransferOperation transfer = await s_defaultTransferManager.Value.StartTransferAsync( blobDirectory, localDirectory, options?.TransferOptions, cancellationToken).ConfigureAwait(false); - if (waitUntil == WaitUntil.Completed) - { - await transfer.WaitForCompletionAsync(cancellationToken).ConfigureAwait(false); - } - - return transfer; + if (waitUntil == WaitUntil.Completed) + { + await transfer.WaitForCompletionAsync(cancellationToken).ConfigureAwait(false); } + + return transfer; } } } diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/PageBlobStartTransferCopyTests.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/PageBlobStartTransferCopyTests.cs index db50d468371e..7a8cb3b7995d 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/PageBlobStartTransferCopyTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/PageBlobStartTransferCopyTests.cs @@ -356,7 +356,7 @@ public async Task CopyPremiumPageBlob_AccessTier(TransferPropertiesTestType prop TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - TransferManager transferManager = new TransferManager(); + await using TransferManager transferManager = new TransferManager(); // Act - Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/ProgressHandlerTests.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/ProgressHandlerTests.cs index 41e49211cf58..d6728c06a339 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/ProgressHandlerTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/ProgressHandlerTests.cs @@ -126,7 +126,7 @@ private async Task TransferAndAssertProgress( ErrorMode = TransferErrorMode.ContinueOnFailure }; - TransferManager transferManager = new TransferManager(transferManagerOptions); + await using TransferManager transferManager = new TransferManager(transferManagerOptions); TestProgressHandler progressHandler = new TestProgressHandler(); transferOptions ??= new TransferOptions(); diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/StartTransferCheckpointerTests.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/StartTransferCheckpointerTests.cs index f398314352cc..5f2a0e70fc31 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/StartTransferCheckpointerTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/StartTransferCheckpointerTests.cs @@ -75,7 +75,7 @@ public async Task CheckpointerWithSasAsync() { CheckpointStoreOptions = TransferCheckpointStoreOptions.CreateLocalStore(disposingLocalDirectory.DirectoryPath) }; - TransferManager transferManager = new TransferManager(managerOptions); + await using TransferManager transferManager = new TransferManager(managerOptions); TransferOptions transferOptions = new TransferOptions() { diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryClientExtensions.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryClientExtensions.cs index b4100b914365..7c80d5b0245c 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryClientExtensions.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryClientExtensions.cs @@ -14,6 +14,8 @@ namespace Azure.Storage.Files.Shares /// public static class ShareDirectoryClientExtensions { + private static Lazy s_defaultTransferManager = new(() => new TransferManager(default)); + /// /// Uploads the entire contents of local directory to the share directory. /// @@ -46,20 +48,17 @@ public static async Task UploadDirectoryAsync( StorageResource localDirectory = LocalFilesStorageResourceProvider.FromDirectory(localDirectoryPath); StorageResource shareDirectory = ShareFilesStorageResourceProvider.FromClient(client, options?.ShareDirectoryOptions); - await using (TransferManager transferManager = new TransferManager()) + TransferOperation trasnfer = await s_defaultTransferManager.Value.StartTransferAsync( + localDirectory, + shareDirectory, + options?.TransferOptions, + cancellationToken).ConfigureAwait(false); + if (waitUntil == WaitUntil.Completed) { - TransferOperation transfer = await transferManager.StartTransferAsync( - localDirectory, - shareDirectory, - options?.TransferOptions, - cancellationToken).ConfigureAwait(false); - if (waitUntil == WaitUntil.Completed) - { - await transfer.WaitForCompletionAsync(cancellationToken).ConfigureAwait(false); - } - - return transfer; + await trasnfer.WaitForCompletionAsync(cancellationToken).ConfigureAwait(false); } + + return trasnfer; } /// @@ -94,20 +93,17 @@ public static async Task DownloadToDirectoryAsync( StorageResource localDirectory = LocalFilesStorageResourceProvider.FromDirectory(localDirectoryPath); StorageResource shareDirectory = ShareFilesStorageResourceProvider.FromClient(client, options?.ShareDirectoryOptions); - await using (TransferManager transferManager = new TransferManager()) + TransferOperation trasnfer = await s_defaultTransferManager.Value.StartTransferAsync( + shareDirectory, + localDirectory, + options?.TransferOptions, + cancellationToken).ConfigureAwait(false); + if (waitUntil == WaitUntil.Completed) { - TransferOperation transfer = await transferManager.StartTransferAsync( - shareDirectory, - localDirectory, - options?.TransferOptions, - cancellationToken).ConfigureAwait(false); - if (waitUntil == WaitUntil.Completed) - { - await transfer.WaitForCompletionAsync(cancellationToken).ConfigureAwait(false); - } - - return transfer; + await trasnfer.WaitForCompletionAsync(cancellationToken).ConfigureAwait(false); } + + return trasnfer; } } } diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs index b76ba3451e7c..74ec4038bb6e 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs @@ -792,7 +792,7 @@ public async Task ShareDirectoryToShareDirectory_PreserveSmb_NoExistingDestNoOve { MaximumConcurrency = 1, }; - TransferManager transferManager = new TransferManager(managerOptions); + await using TransferManager transferManager = new TransferManager(managerOptions); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -886,7 +886,7 @@ public async Task ShareDirectoryToShareDirectory_PreserveSmb_ExistingDestOverwri { MaximumConcurrency = 1, }; - TransferManager transferManager = new TransferManager(managerOptions); + await using TransferManager transferManager = new TransferManager(managerOptions); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -980,7 +980,7 @@ public async Task ShareDirectoryToShareDirectory_PreserveSmb_ExistingDestNoOverw { MaximumConcurrency = 1, }; - TransferManager transferManager = new TransferManager(managerOptions); + await using TransferManager transferManager = new TransferManager(managerOptions); // Act and Assert TransferOperation transfer = await transferManager.StartTransferAsync(sourceResource, destinationResource, options); @@ -1044,7 +1044,7 @@ public async Task ShareDirectoryToShareDirectory_PreserveSmb_NoExistingDestOverw { MaximumConcurrency = 1, }; - TransferManager transferManager = new TransferManager(managerOptions); + await using TransferManager transferManager = new TransferManager(managerOptions); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -1126,7 +1126,7 @@ public async Task ShareDirectoryToShareDirectory_PreserveNfs_NoExistingDestNoOve { MaximumConcurrency = 1, }; - TransferManager transferManager = new TransferManager(managerOptions); + await using TransferManager transferManager = new TransferManager(managerOptions); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -1226,7 +1226,7 @@ public async Task ShareDirectoryToShareDirectory_PreserveNfs_ExistingDestOverwri { MaximumConcurrency = 1, }; - TransferManager transferManager = new TransferManager(managerOptions); + await using TransferManager transferManager = new TransferManager(managerOptions); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -1326,7 +1326,7 @@ public async Task ShareDirectoryToShareDirectory_PreserveNfs_ExistingDestNoOverw { MaximumConcurrency = 1, }; - TransferManager transferManager = new TransferManager(managerOptions); + await using TransferManager transferManager = new TransferManager(managerOptions); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -1398,7 +1398,7 @@ public async Task ShareDirectoryToShareDirectory_PreserveNfs_NoExistingDestOverw { MaximumConcurrency = 1, }; - TransferManager transferManager = new TransferManager(managerOptions); + await using TransferManager transferManager = new TransferManager(managerOptions); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -1484,7 +1484,7 @@ public async Task ShareDirectoryToShareDirectory_SmbDestinationOptionsOverride() TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); TransferManagerOptions managerOptions = new TransferManagerOptions() { MaximumConcurrency = 1 }; - TransferManager transferManager = new TransferManager(managerOptions); + await using TransferManager transferManager = new TransferManager(managerOptions); TransferOperation transfer = await transferManager.StartTransferAsync( sourceResource, @@ -1577,7 +1577,7 @@ public async Task ShareDirectoryToShareDirectory_NfsDestinationOptionsOverride(b TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); TransferManagerOptions managerOptions = new TransferManagerOptions() { MaximumConcurrency = 1 }; - TransferManager transferManager = new TransferManager(managerOptions); + await using TransferManager transferManager = new TransferManager(managerOptions); TransferOperation transfer = await transferManager.StartTransferAsync( sourceResource, @@ -1672,7 +1672,7 @@ public async Task ValidateProtocolAsync_SmbShareDirectoryToSmbShareDirectory_Com { MaximumConcurrency = 1, }; - TransferManager transferManager = new TransferManager(managerOptions); + await using TransferManager transferManager = new TransferManager(managerOptions); // Act and Assert if (sourceProtocol == ShareProtocol.Smb && destProtocol == ShareProtocol.Smb) @@ -1760,7 +1760,7 @@ public async Task ValidateProtocolAsync_NfsShareDirectoryToNfsShareDirectory_Com { MaximumConcurrency = 1, }; - TransferManager transferManager = new TransferManager(managerOptions); + await using TransferManager transferManager = new TransferManager(managerOptions); // Act and Assert if (sourceProtocol == ShareProtocol.Nfs && destProtocol == ShareProtocol.Nfs) @@ -1848,7 +1848,7 @@ public async Task ShareDirectoryToShareDirectory_NfsHardLink() { MaximumConcurrency = 1, }; - TransferManager transferManager = new TransferManager(managerOptions); + await using TransferManager transferManager = new TransferManager(managerOptions); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -1917,7 +1917,7 @@ public async Task ShareDirectoryToShareDirectory_NfsSymbolicLink() { MaximumConcurrency = 1, }; - TransferManager transferManager = new TransferManager(managerOptions); + await using TransferManager transferManager = new TransferManager(managerOptions); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -2016,7 +2016,7 @@ public async Task ShareDirectoryNfsToShareDirectorySmb(bool? filePermissions) // Create Transfer Manager with single threaded operation TransferManagerOptions managerOptions = new TransferManagerOptions(){ MaximumConcurrency = 1 }; - TransferManager transferManager = new TransferManager(managerOptions); + await using TransferManager transferManager = new TransferManager(managerOptions); if (filePermissions == true) { @@ -2098,7 +2098,7 @@ public async Task ShareDirectorySmbToShareDirectoryNfs(bool? filePermissions) // Create Transfer Manager with single threaded operation TransferManagerOptions managerOptions = new TransferManagerOptions(){ MaximumConcurrency = 1 }; - TransferManager transferManager = new TransferManager(managerOptions); + await using TransferManager transferManager = new TransferManager(managerOptions); if (filePermissions == true) { @@ -2186,7 +2186,7 @@ public async Task ShareDirectoryToShareDirectory_PreserveNfs_OAuth(bool? filePer { MaximumConcurrency = 1, }; - TransferManager transferManager = new TransferManager(managerOptions); + await using TransferManager transferManager = new TransferManager(managerOptions); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferDownloadTests.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferDownloadTests.cs index d4834ef36e91..b95d6b38d351 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferDownloadTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferDownloadTests.cs @@ -221,7 +221,7 @@ public async Task ShareDirectoryToLocalDirectory_NfsHardLink() { MaximumConcurrency = 1, }; - TransferManager transferManager = new TransferManager(managerOptions); + await using TransferManager transferManager = new TransferManager(managerOptions); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -289,7 +289,7 @@ public async Task ShareDirectoryToLocalDirectory_NfsSymbolicLink() { MaximumConcurrency = 1, }; - TransferManager transferManager = new TransferManager(managerOptions); + await using TransferManager transferManager = new TransferManager(managerOptions); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareFileStartTransferCopyTests.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareFileStartTransferCopyTests.cs index 03e5bdb389eb..878b48afebe1 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareFileStartTransferCopyTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareFileStartTransferCopyTests.cs @@ -480,7 +480,7 @@ private async Task CopyRemoteObjects_VerifyProperties( TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - TransferManager transferManager = new TransferManager(); + await using TransferManager transferManager = new TransferManager(); // Act - Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -537,7 +537,7 @@ public async Task ShareFileToShareFile_ManuallySetFileAttributes( TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - TransferManager transferManager = new TransferManager(); + await using TransferManager transferManager = new TransferManager(); // Act - Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -592,7 +592,7 @@ public async Task ShareFileToShareFile_PreserveFromSourceFileAttributes( TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - TransferManager transferManager = new TransferManager(); + await using TransferManager transferManager = new TransferManager(); // Act - Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -680,7 +680,7 @@ public async Task ShareFileToShareFile_PermissionValue(TransferPropertiesTestTyp TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - TransferManager transferManager = new TransferManager(); + await using TransferManager transferManager = new TransferManager(); // Act - Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -814,7 +814,7 @@ public async Task ShareFileToShareFile_PreserveSmb_OverwriteExistingDest(bool? f TransferOptions options = new TransferOptions() { CreationMode = StorageResourceCreationMode.OverwriteIfExists }; TestEventsRaised testEventsRaised = new TestEventsRaised(options); - TransferManager transferManager = new TransferManager(); + await using TransferManager transferManager = new TransferManager(); // Act - Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -934,7 +934,7 @@ public async Task ShareFileToShareFile_PreserveNfs(bool? filePermissions) TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - TransferManager transferManager = new TransferManager(); + await using TransferManager transferManager = new TransferManager(); // Act - Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -1061,7 +1061,7 @@ public async Task ShareFileToShareFile_PreserveNfs_OverwriteExistingDest(bool? f TransferOptions options = new TransferOptions() { CreationMode = StorageResourceCreationMode.OverwriteIfExists }; TestEventsRaised testEventsRaised = new TestEventsRaised(options); - TransferManager transferManager = new TransferManager(); + await using TransferManager transferManager = new TransferManager(); // Act - Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -1178,7 +1178,7 @@ public async Task ShareFileToShareFile_NfsDestinationOptionsOverride(bool? fileP TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - TransferManager transferManager = new TransferManager(); + await using TransferManager transferManager = new TransferManager(); // Act - Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -1253,7 +1253,7 @@ public async Task ValidateProtocolAsync_SmbShareFileToSmbShareFile_CompareProtoc TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - TransferManager transferManager = new TransferManager(); + await using TransferManager transferManager = new TransferManager(); // Act and Assert if (sourceProtocol == ShareProtocol.Smb && destProtocol == ShareProtocol.Smb) @@ -1312,7 +1312,7 @@ public async Task ValidateProtocolAsync_NfsShareFileToNfsShareFile_CompareProtoc TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - TransferManager transferManager = new TransferManager(); + await using TransferManager transferManager = new TransferManager(); // Act and Assert if (sourceProtocol == ShareProtocol.Nfs && destProtocol == ShareProtocol.Nfs) @@ -1371,7 +1371,7 @@ public async Task ValidateProtocolAsync_NoShareLevelPermissions_SkipProtocolVali TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - TransferManager transferManager = new TransferManager(); + await using TransferManager transferManager = new TransferManager(); // Act and Assert if (skipProtocolValidation) @@ -1462,7 +1462,7 @@ await hardlinkClient.CreateHardLinkAsync( TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - TransferManager transferManager = new TransferManager(); + await using TransferManager transferManager = new TransferManager(); // Act - Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -1532,7 +1532,7 @@ public async Task ShareFileToShareFile_NfsSymbolicLink() TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - TransferManager transferManager = new TransferManager(); + await using TransferManager transferManager = new TransferManager(); // Act - Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -1606,7 +1606,7 @@ public async Task ShareFileNfsToShareFileSmb(bool? filePermissions) TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - TransferManager transferManager = new TransferManager(); + await using TransferManager transferManager = new TransferManager(); if (filePermissions == true) { @@ -1694,7 +1694,7 @@ public async Task ShareFileSmbToShareFileNfs(bool? filePermissions) TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - TransferManager transferManager = new TransferManager(); + await using TransferManager transferManager = new TransferManager(); if (filePermissions == true) { @@ -1786,7 +1786,7 @@ public async Task ShareFileToShareFile_PreserveNfs_OAuth(bool? filePermissions) TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - TransferManager transferManager = new TransferManager(); + await using TransferManager transferManager = new TransferManager(); // Act - Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareFileStartTransferDownloadTests.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareFileStartTransferDownloadTests.cs index 8243905a8fb3..96ed34a82f2e 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareFileStartTransferDownloadTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareFileStartTransferDownloadTests.cs @@ -127,7 +127,7 @@ await hardlinkClient.CreateHardLinkAsync( TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - TransferManager transferManager = new TransferManager(); + await using TransferManager transferManager = new TransferManager(); // Act - Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -188,7 +188,7 @@ public async Task ShareFileToLocalFile_NfsSymbolicLink() TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - TransferManager transferManager = new TransferManager(); + await using TransferManager transferManager = new TransferManager(); // Act - Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferUploadTestBase.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferUploadTestBase.cs index dfad36c6d729..e46bae110487 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferUploadTestBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferUploadTestBase.cs @@ -568,7 +568,7 @@ public async Task LocalToRemoteObject_Skip_Exists() StorageResourceItem destinationResource = GetStorageResourceItem(objectClient); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - TransferManager transferManager = new TransferManager(); + await using TransferManager transferManager = new TransferManager(); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -625,7 +625,7 @@ public async Task LocalToRemoteObject_Failure_Exists() TestEventsRaised testEventRaised = new TestEventsRaised(options); StorageResource sourceResource = LocalFilesStorageResourceProvider.FromFile(newSourceFile); StorageResourceItem destinationResource = GetStorageResourceItem(objectClient); - TransferManager transferManager = new TransferManager(); + await using TransferManager transferManager = new TransferManager(); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( From afbb22ca52e189844f89e15a885465c980ea87e4 Mon Sep 17 00:00:00 2001 From: Amanda Nguyen Date: Thu, 2 Oct 2025 14:54:14 -0700 Subject: [PATCH 5/8] Undo change to changelog --- sdk/storage/Azure.Storage.DataMovement.Blobs/CHANGELOG.md | 1 - sdk/storage/Azure.Storage.DataMovement.Files.Shares/CHANGELOG.md | 1 - 2 files changed, 2 deletions(-) diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/CHANGELOG.md b/sdk/storage/Azure.Storage.DataMovement.Blobs/CHANGELOG.md index b44b2c8d29f8..bd45cc443930 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/CHANGELOG.md +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/CHANGELOG.md @@ -11,7 +11,6 @@ - `TransferOperation` disposes the `CancellationTokenSource` after transfer reaches a `Completed` or `Paused` state - `TransferManager` uses a `CancellationTokenSource` also does not link the`CancellationToken` passed to it's methods - Removed usage of `CancellationTokenSource` from handling the chunking of large transfers. This only affects transfers that cannot be completed in one request. - - Disposed internal use of `TransferManager` in BlobContainerClient extension methods, `UploadDirectoryAsync(..)` and `DownloadToDirectoryAsync(..)` - Fixed bug where cached referenced `TransferOperation`s from the `TransferManager` were not being cleared on dispose. - Fixed bug where referenced `TransferOperation` from the transfers stored in the `TransferManager` after they reach a `Completed` or `Paused` state where not being removed. diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/CHANGELOG.md b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/CHANGELOG.md index c2abbce045bb..b3f8522d50b3 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/CHANGELOG.md +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/CHANGELOG.md @@ -11,7 +11,6 @@ - `TransferOperation` disposes the `CancellationTokenSource` after transfer reaches a `Completed` or `Paused` state - `TransferManager` uses a `CancellationTokenSource` also does not link the`CancellationToken` passed to it's methods - Removed usage of `CancellationTokenSource` from handling the chunking of large transfers. This only affects transfers that cannot be completed in one request. - - Disposed internal use of `TransferManager` in ShareDirectoryClient extension methods, `UploadDirectoryAsync(..)` and `DownloadToDirectoryAsync(..)` - Fixed bug where cached referenced `TransferOperation`s from the `TransferManager` were not being cleared on dispose. - Fixed bug where referenced `TransferOperation` from the transfers stored in the `TransferManager` after they reach a `Completed` or `Paused` state where not being removed. From f2905d8ceeb9105183e86137395a6dd0631e6124 Mon Sep 17 00:00:00 2001 From: Amanda Nguyen Date: Thu, 2 Oct 2025 14:56:28 -0700 Subject: [PATCH 6/8] Update Snippets --- .../Azure.Storage.DataMovement/MigrationGuide.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sdk/storage/Azure.Storage.DataMovement/MigrationGuide.md b/sdk/storage/Azure.Storage.DataMovement/MigrationGuide.md index 6d0b45c10ad4..d8c3a3340ec1 100644 --- a/sdk/storage/Azure.Storage.DataMovement/MigrationGuide.md +++ b/sdk/storage/Azure.Storage.DataMovement/MigrationGuide.md @@ -186,7 +186,7 @@ await TransferManager.UploadAsync( string filePath; Uri blobUri; BlobsStorageResourceProvider blobs; -TransferManager transferManager; +await using TransferManager transferManager = new TransferManager(); ``` ```C# Snippet:DataMovementMigration_UploadSingleFile // upload blob @@ -216,7 +216,7 @@ await TransferManager.UploadDirectoryAsync( string directoryPath, blobDirectoryPath; Uri containerUri; BlobsStorageResourceProvider blobs; -TransferManager transferManager; +await using TransferManager transferManager = new TransferManager(); ``` ```C# Snippet:DataMovementMigration_UploadBlobDirectory // upload blobs @@ -251,7 +251,7 @@ await TransferManager.DownloadAsync( string filePath; Uri blobUri; BlobsStorageResourceProvider blobs; -TransferManager transferManager; +await using TransferManager transferManager = new TransferManager(); ``` ```C# Snippet:DataMovementMigration_DownloadBlob // download blob @@ -283,7 +283,7 @@ await TransferManager.DownloadDirectoryAsync( string directoryPath, blobDirectoryPath; Uri containerUri; BlobsStorageResourceProvider blobs; -TransferManager transferManager; +await using TransferManager transferManager = new TransferManager(); ``` ```C# Snippet:DataMovementMigration_DownloadBlobDirectory // download blob directory @@ -320,7 +320,7 @@ await TransferManager.DownloadAsync( // these values provided by your code Uri srcBlobUri, dstBlobUri; BlobsStorageResourceProvider blobs; -TransferManager transferManager; +await using TransferManager transferManager = new TransferManager(); ``` ```C# Snippet:DataMovementMigration_CopyBlobToBlob // upload blob @@ -354,7 +354,7 @@ await TransferManager.DownloadAsync( Uri blobUri, fileUri; BlobsStorageResourceProvider blobs; ShareFilesStorageResourceProvider files; -TransferManager transferManager; +await using TransferManager transferManager = new TransferManager(); ``` ```C# Snippet:DataMovementMigration_CopyBlobToShareFile // upload blob From 7cfaeb50cf7f177e8b6345db6da6e196432dc080 Mon Sep 17 00:00:00 2001 From: Amanda Nguyen Date: Mon, 6 Oct 2025 16:11:25 -0700 Subject: [PATCH 7/8] Undo transfermanager dispose test changes --- .../tests/PageBlobStartTransferCopyTests.cs | 2 +- .../tests/ProgressHandlerTests.cs | 2 +- .../tests/StartTransferCheckpointerTests.cs | 2 +- .../ShareDirectoryStartTransferCopyTests.cs | 34 ++--- ...hareDirectoryStartTransferDownloadTests.cs | 4 +- .../tests/ShareFileStartTransferCopyTests.cs | 32 ++--- .../ShareFileStartTransferDownloadTests.cs | 4 +- .../MigrationGuide.md | 12 +- .../samples/Sample01b_Migration.cs | 18 ++- .../tests/GetTransfersTests.cs | 16 +-- .../Shared/PauseResumeTransferTestBase.cs | 28 ++-- .../tests/Shared/StartTransferCopyTestBase.cs | 40 +++--- .../StartTransferDirectoryCopyTestBase.cs | 51 +++---- .../StartTransferDirectoryDownloadTestBase.cs | 47 ++----- .../Shared/StartTransferDownloadTestBase.cs | 47 +++---- .../StartTransferUploadDirectoryTestBase.cs | 127 +++++++----------- .../Shared/StartTransferUploadTestBase.cs | 44 ++---- .../tests/TransferOperationTests.cs | 3 +- 18 files changed, 205 insertions(+), 308 deletions(-) diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/PageBlobStartTransferCopyTests.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/PageBlobStartTransferCopyTests.cs index 7a8cb3b7995d..db50d468371e 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/PageBlobStartTransferCopyTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/PageBlobStartTransferCopyTests.cs @@ -356,7 +356,7 @@ public async Task CopyPremiumPageBlob_AccessTier(TransferPropertiesTestType prop TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager = new TransferManager(); // Act - Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/ProgressHandlerTests.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/ProgressHandlerTests.cs index d6728c06a339..41e49211cf58 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/ProgressHandlerTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/ProgressHandlerTests.cs @@ -126,7 +126,7 @@ private async Task TransferAndAssertProgress( ErrorMode = TransferErrorMode.ContinueOnFailure }; - await using TransferManager transferManager = new TransferManager(transferManagerOptions); + TransferManager transferManager = new TransferManager(transferManagerOptions); TestProgressHandler progressHandler = new TestProgressHandler(); transferOptions ??= new TransferOptions(); diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/StartTransferCheckpointerTests.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/StartTransferCheckpointerTests.cs index 5f2a0e70fc31..f398314352cc 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/StartTransferCheckpointerTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/StartTransferCheckpointerTests.cs @@ -75,7 +75,7 @@ public async Task CheckpointerWithSasAsync() { CheckpointStoreOptions = TransferCheckpointStoreOptions.CreateLocalStore(disposingLocalDirectory.DirectoryPath) }; - await using TransferManager transferManager = new TransferManager(managerOptions); + TransferManager transferManager = new TransferManager(managerOptions); TransferOptions transferOptions = new TransferOptions() { diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs index 74ec4038bb6e..b76ba3451e7c 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs @@ -792,7 +792,7 @@ public async Task ShareDirectoryToShareDirectory_PreserveSmb_NoExistingDestNoOve { MaximumConcurrency = 1, }; - await using TransferManager transferManager = new TransferManager(managerOptions); + TransferManager transferManager = new TransferManager(managerOptions); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -886,7 +886,7 @@ public async Task ShareDirectoryToShareDirectory_PreserveSmb_ExistingDestOverwri { MaximumConcurrency = 1, }; - await using TransferManager transferManager = new TransferManager(managerOptions); + TransferManager transferManager = new TransferManager(managerOptions); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -980,7 +980,7 @@ public async Task ShareDirectoryToShareDirectory_PreserveSmb_ExistingDestNoOverw { MaximumConcurrency = 1, }; - await using TransferManager transferManager = new TransferManager(managerOptions); + TransferManager transferManager = new TransferManager(managerOptions); // Act and Assert TransferOperation transfer = await transferManager.StartTransferAsync(sourceResource, destinationResource, options); @@ -1044,7 +1044,7 @@ public async Task ShareDirectoryToShareDirectory_PreserveSmb_NoExistingDestOverw { MaximumConcurrency = 1, }; - await using TransferManager transferManager = new TransferManager(managerOptions); + TransferManager transferManager = new TransferManager(managerOptions); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -1126,7 +1126,7 @@ public async Task ShareDirectoryToShareDirectory_PreserveNfs_NoExistingDestNoOve { MaximumConcurrency = 1, }; - await using TransferManager transferManager = new TransferManager(managerOptions); + TransferManager transferManager = new TransferManager(managerOptions); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -1226,7 +1226,7 @@ public async Task ShareDirectoryToShareDirectory_PreserveNfs_ExistingDestOverwri { MaximumConcurrency = 1, }; - await using TransferManager transferManager = new TransferManager(managerOptions); + TransferManager transferManager = new TransferManager(managerOptions); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -1326,7 +1326,7 @@ public async Task ShareDirectoryToShareDirectory_PreserveNfs_ExistingDestNoOverw { MaximumConcurrency = 1, }; - await using TransferManager transferManager = new TransferManager(managerOptions); + TransferManager transferManager = new TransferManager(managerOptions); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -1398,7 +1398,7 @@ public async Task ShareDirectoryToShareDirectory_PreserveNfs_NoExistingDestOverw { MaximumConcurrency = 1, }; - await using TransferManager transferManager = new TransferManager(managerOptions); + TransferManager transferManager = new TransferManager(managerOptions); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -1484,7 +1484,7 @@ public async Task ShareDirectoryToShareDirectory_SmbDestinationOptionsOverride() TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); TransferManagerOptions managerOptions = new TransferManagerOptions() { MaximumConcurrency = 1 }; - await using TransferManager transferManager = new TransferManager(managerOptions); + TransferManager transferManager = new TransferManager(managerOptions); TransferOperation transfer = await transferManager.StartTransferAsync( sourceResource, @@ -1577,7 +1577,7 @@ public async Task ShareDirectoryToShareDirectory_NfsDestinationOptionsOverride(b TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); TransferManagerOptions managerOptions = new TransferManagerOptions() { MaximumConcurrency = 1 }; - await using TransferManager transferManager = new TransferManager(managerOptions); + TransferManager transferManager = new TransferManager(managerOptions); TransferOperation transfer = await transferManager.StartTransferAsync( sourceResource, @@ -1672,7 +1672,7 @@ public async Task ValidateProtocolAsync_SmbShareDirectoryToSmbShareDirectory_Com { MaximumConcurrency = 1, }; - await using TransferManager transferManager = new TransferManager(managerOptions); + TransferManager transferManager = new TransferManager(managerOptions); // Act and Assert if (sourceProtocol == ShareProtocol.Smb && destProtocol == ShareProtocol.Smb) @@ -1760,7 +1760,7 @@ public async Task ValidateProtocolAsync_NfsShareDirectoryToNfsShareDirectory_Com { MaximumConcurrency = 1, }; - await using TransferManager transferManager = new TransferManager(managerOptions); + TransferManager transferManager = new TransferManager(managerOptions); // Act and Assert if (sourceProtocol == ShareProtocol.Nfs && destProtocol == ShareProtocol.Nfs) @@ -1848,7 +1848,7 @@ public async Task ShareDirectoryToShareDirectory_NfsHardLink() { MaximumConcurrency = 1, }; - await using TransferManager transferManager = new TransferManager(managerOptions); + TransferManager transferManager = new TransferManager(managerOptions); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -1917,7 +1917,7 @@ public async Task ShareDirectoryToShareDirectory_NfsSymbolicLink() { MaximumConcurrency = 1, }; - await using TransferManager transferManager = new TransferManager(managerOptions); + TransferManager transferManager = new TransferManager(managerOptions); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -2016,7 +2016,7 @@ public async Task ShareDirectoryNfsToShareDirectorySmb(bool? filePermissions) // Create Transfer Manager with single threaded operation TransferManagerOptions managerOptions = new TransferManagerOptions(){ MaximumConcurrency = 1 }; - await using TransferManager transferManager = new TransferManager(managerOptions); + TransferManager transferManager = new TransferManager(managerOptions); if (filePermissions == true) { @@ -2098,7 +2098,7 @@ public async Task ShareDirectorySmbToShareDirectoryNfs(bool? filePermissions) // Create Transfer Manager with single threaded operation TransferManagerOptions managerOptions = new TransferManagerOptions(){ MaximumConcurrency = 1 }; - await using TransferManager transferManager = new TransferManager(managerOptions); + TransferManager transferManager = new TransferManager(managerOptions); if (filePermissions == true) { @@ -2186,7 +2186,7 @@ public async Task ShareDirectoryToShareDirectory_PreserveNfs_OAuth(bool? filePer { MaximumConcurrency = 1, }; - await using TransferManager transferManager = new TransferManager(managerOptions); + TransferManager transferManager = new TransferManager(managerOptions); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferDownloadTests.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferDownloadTests.cs index b95d6b38d351..d4834ef36e91 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferDownloadTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferDownloadTests.cs @@ -221,7 +221,7 @@ public async Task ShareDirectoryToLocalDirectory_NfsHardLink() { MaximumConcurrency = 1, }; - await using TransferManager transferManager = new TransferManager(managerOptions); + TransferManager transferManager = new TransferManager(managerOptions); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -289,7 +289,7 @@ public async Task ShareDirectoryToLocalDirectory_NfsSymbolicLink() { MaximumConcurrency = 1, }; - await using TransferManager transferManager = new TransferManager(managerOptions); + TransferManager transferManager = new TransferManager(managerOptions); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareFileStartTransferCopyTests.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareFileStartTransferCopyTests.cs index 878b48afebe1..03e5bdb389eb 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareFileStartTransferCopyTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareFileStartTransferCopyTests.cs @@ -480,7 +480,7 @@ private async Task CopyRemoteObjects_VerifyProperties( TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager = new TransferManager(); // Act - Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -537,7 +537,7 @@ public async Task ShareFileToShareFile_ManuallySetFileAttributes( TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager = new TransferManager(); // Act - Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -592,7 +592,7 @@ public async Task ShareFileToShareFile_PreserveFromSourceFileAttributes( TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager = new TransferManager(); // Act - Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -680,7 +680,7 @@ public async Task ShareFileToShareFile_PermissionValue(TransferPropertiesTestTyp TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager = new TransferManager(); // Act - Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -814,7 +814,7 @@ public async Task ShareFileToShareFile_PreserveSmb_OverwriteExistingDest(bool? f TransferOptions options = new TransferOptions() { CreationMode = StorageResourceCreationMode.OverwriteIfExists }; TestEventsRaised testEventsRaised = new TestEventsRaised(options); - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager = new TransferManager(); // Act - Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -934,7 +934,7 @@ public async Task ShareFileToShareFile_PreserveNfs(bool? filePermissions) TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager = new TransferManager(); // Act - Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -1061,7 +1061,7 @@ public async Task ShareFileToShareFile_PreserveNfs_OverwriteExistingDest(bool? f TransferOptions options = new TransferOptions() { CreationMode = StorageResourceCreationMode.OverwriteIfExists }; TestEventsRaised testEventsRaised = new TestEventsRaised(options); - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager = new TransferManager(); // Act - Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -1178,7 +1178,7 @@ public async Task ShareFileToShareFile_NfsDestinationOptionsOverride(bool? fileP TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager = new TransferManager(); // Act - Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -1253,7 +1253,7 @@ public async Task ValidateProtocolAsync_SmbShareFileToSmbShareFile_CompareProtoc TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager = new TransferManager(); // Act and Assert if (sourceProtocol == ShareProtocol.Smb && destProtocol == ShareProtocol.Smb) @@ -1312,7 +1312,7 @@ public async Task ValidateProtocolAsync_NfsShareFileToNfsShareFile_CompareProtoc TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager = new TransferManager(); // Act and Assert if (sourceProtocol == ShareProtocol.Nfs && destProtocol == ShareProtocol.Nfs) @@ -1371,7 +1371,7 @@ public async Task ValidateProtocolAsync_NoShareLevelPermissions_SkipProtocolVali TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager = new TransferManager(); // Act and Assert if (skipProtocolValidation) @@ -1462,7 +1462,7 @@ await hardlinkClient.CreateHardLinkAsync( TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager = new TransferManager(); // Act - Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -1532,7 +1532,7 @@ public async Task ShareFileToShareFile_NfsSymbolicLink() TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager = new TransferManager(); // Act - Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -1606,7 +1606,7 @@ public async Task ShareFileNfsToShareFileSmb(bool? filePermissions) TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager = new TransferManager(); if (filePermissions == true) { @@ -1694,7 +1694,7 @@ public async Task ShareFileSmbToShareFileNfs(bool? filePermissions) TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager = new TransferManager(); if (filePermissions == true) { @@ -1786,7 +1786,7 @@ public async Task ShareFileToShareFile_PreserveNfs_OAuth(bool? filePermissions) TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager = new TransferManager(); // Act - Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareFileStartTransferDownloadTests.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareFileStartTransferDownloadTests.cs index 96ed34a82f2e..8243905a8fb3 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareFileStartTransferDownloadTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareFileStartTransferDownloadTests.cs @@ -127,7 +127,7 @@ await hardlinkClient.CreateHardLinkAsync( TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager = new TransferManager(); // Act - Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -188,7 +188,7 @@ public async Task ShareFileToLocalFile_NfsSymbolicLink() TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager = new TransferManager(); // Act - Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( diff --git a/sdk/storage/Azure.Storage.DataMovement/MigrationGuide.md b/sdk/storage/Azure.Storage.DataMovement/MigrationGuide.md index d8c3a3340ec1..6d0b45c10ad4 100644 --- a/sdk/storage/Azure.Storage.DataMovement/MigrationGuide.md +++ b/sdk/storage/Azure.Storage.DataMovement/MigrationGuide.md @@ -186,7 +186,7 @@ await TransferManager.UploadAsync( string filePath; Uri blobUri; BlobsStorageResourceProvider blobs; -await using TransferManager transferManager = new TransferManager(); +TransferManager transferManager; ``` ```C# Snippet:DataMovementMigration_UploadSingleFile // upload blob @@ -216,7 +216,7 @@ await TransferManager.UploadDirectoryAsync( string directoryPath, blobDirectoryPath; Uri containerUri; BlobsStorageResourceProvider blobs; -await using TransferManager transferManager = new TransferManager(); +TransferManager transferManager; ``` ```C# Snippet:DataMovementMigration_UploadBlobDirectory // upload blobs @@ -251,7 +251,7 @@ await TransferManager.DownloadAsync( string filePath; Uri blobUri; BlobsStorageResourceProvider blobs; -await using TransferManager transferManager = new TransferManager(); +TransferManager transferManager; ``` ```C# Snippet:DataMovementMigration_DownloadBlob // download blob @@ -283,7 +283,7 @@ await TransferManager.DownloadDirectoryAsync( string directoryPath, blobDirectoryPath; Uri containerUri; BlobsStorageResourceProvider blobs; -await using TransferManager transferManager = new TransferManager(); +TransferManager transferManager; ``` ```C# Snippet:DataMovementMigration_DownloadBlobDirectory // download blob directory @@ -320,7 +320,7 @@ await TransferManager.DownloadAsync( // these values provided by your code Uri srcBlobUri, dstBlobUri; BlobsStorageResourceProvider blobs; -await using TransferManager transferManager = new TransferManager(); +TransferManager transferManager; ``` ```C# Snippet:DataMovementMigration_CopyBlobToBlob // upload blob @@ -354,7 +354,7 @@ await TransferManager.DownloadAsync( Uri blobUri, fileUri; BlobsStorageResourceProvider blobs; ShareFilesStorageResourceProvider files; -await using TransferManager transferManager = new TransferManager(); +TransferManager transferManager; ``` ```C# Snippet:DataMovementMigration_CopyBlobToShareFile // upload blob diff --git a/sdk/storage/Azure.Storage.DataMovement/samples/Sample01b_Migration.cs b/sdk/storage/Azure.Storage.DataMovement/samples/Sample01b_Migration.cs index f95d61599ee0..3b11393a2b33 100644 --- a/sdk/storage/Azure.Storage.DataMovement/samples/Sample01b_Migration.cs +++ b/sdk/storage/Azure.Storage.DataMovement/samples/Sample01b_Migration.cs @@ -53,7 +53,7 @@ public async Task UploadBlob() string filePath; Uri blobUri; BlobsStorageResourceProvider blobs; - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager; #endregion // Create a temporary Lorem Ipsum file on disk to upload @@ -79,6 +79,7 @@ public async Task UploadBlob() try { + transferManager = new TransferManager(); blobs = new BlobsStorageResourceProvider(credential); #region Snippet:DataMovementMigration_UploadSingleFile @@ -103,7 +104,7 @@ public async Task UploadBlobDirectory() string directoryPath, blobDirectoryPath; Uri containerUri; BlobsStorageResourceProvider blobs; - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager; #endregion // Create a temporary populated directory on disk to upload @@ -128,6 +129,7 @@ public async Task UploadBlobDirectory() try { + transferManager = new TransferManager(); blobs = new BlobsStorageResourceProvider(credential); #region Snippet:DataMovementMigration_UploadBlobDirectory @@ -155,7 +157,7 @@ public async Task DownloadBlob() string filePath; Uri blobUri; BlobsStorageResourceProvider blobs; - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager; #endregion // Create a temporary populated directory on disk to upload @@ -184,6 +186,7 @@ public async Task DownloadBlob() await container.GetBlobClient(blobName) .UploadAsync(BinaryData.FromString(SampleFileContent)); + transferManager = new TransferManager(); blobs = new BlobsStorageResourceProvider(credential); #region Snippet:DataMovementMigration_DownloadBlob @@ -208,7 +211,7 @@ public async Task DownloadBlobDirectory() string directoryPath, blobDirectoryPath; Uri containerUri; BlobsStorageResourceProvider blobs; - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager; #endregion // Create a temporary populated directory on disk to upload @@ -233,6 +236,7 @@ public async Task DownloadBlobDirectory() try { + transferManager = new TransferManager(); blobs = new BlobsStorageResourceProvider(credential); #region Snippet:DataMovementMigration_DownloadBlobDirectory @@ -259,7 +263,7 @@ public async Task CopyBlob() // these values provided by your code Uri srcBlobUri, dstBlobUri; BlobsStorageResourceProvider blobs; - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager; #endregion // Get account and shared key access @@ -291,6 +295,7 @@ public async Task CopyBlob() await container.GetBlobClient(srcBlobName) .UploadAsync(BinaryData.FromString(SampleFileContent)); + transferManager = new TransferManager(); blobs = new BlobsStorageResourceProvider(credential); #region Snippet:DataMovementMigration_CopyBlobToBlob @@ -315,7 +320,7 @@ public async Task CopyBlobToShare() Uri blobUri, fileUri; BlobsStorageResourceProvider blobs; ShareFilesStorageResourceProvider files; - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager; #endregion // Get account and shared key access @@ -352,6 +357,7 @@ await container.GetBlobClient(blobName) share = new ShareClient(accountUri, credential); await share.CreateIfNotExistsAsync(); + transferManager = new TransferManager(); blobs = new BlobsStorageResourceProvider(credential); files = new ShareFilesStorageResourceProvider(credential); diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/GetTransfersTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/GetTransfersTests.cs index 5f8ee21d41d5..9522472af77f 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/GetTransfersTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/GetTransfersTests.cs @@ -58,7 +58,7 @@ public async Task GetTransfers_Empty() { using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); // Arrange - await using TransferManager manager = new TransferManager(GetDefaultManagerOptions(testDirectory.DirectoryPath)); + TransferManager manager = new TransferManager(GetDefaultManagerOptions(testDirectory.DirectoryPath)); // Act IList transfers = await manager.GetTransfersAsync().ToListAsync(); @@ -80,7 +80,7 @@ public async Task GetTransfers_Populated() }; TransferManagerFactory factory = new TransferManagerFactory(GetDefaultManagerOptions(testDirectory.DirectoryPath)); - await using TransferManager manager = factory.BuildTransferManager(storedTransfers); + TransferManager manager = factory.BuildTransferManager(storedTransfers); // Act IList result = await manager.GetTransfersAsync().ToListAsync(); @@ -120,7 +120,7 @@ public async Task GetTransfers_Filtered( }; TransferManagerFactory factory = new TransferManagerFactory(GetDefaultManagerOptions(testDirectory.DirectoryPath)); - await using TransferManager manager = factory.BuildTransferManager(storedTransfers); + TransferManager manager = factory.BuildTransferManager(storedTransfers); // Act TransferStatus[] status = { new TransferStatus(state, hasFailedItems, hasSkippedItems) }; @@ -154,7 +154,7 @@ public async Task GetTransfers_FilterMultipleStatuses() }; TransferManagerFactory factory = new TransferManagerFactory(GetDefaultManagerOptions(testDirectory.DirectoryPath)); - await using TransferManager manager = factory.BuildTransferManager(storedTransfers); + TransferManager manager = factory.BuildTransferManager(storedTransfers); // Act TransferStatus[] statuses = new TransferStatus[] { @@ -190,7 +190,7 @@ public async Task GetTransfers_Filtered_Empty() }; TransferManagerFactory factory = new TransferManagerFactory(GetDefaultManagerOptions(testDirectory.DirectoryPath)); - await using TransferManager manager = factory.BuildTransferManager(storedTransfers); + TransferManager manager = factory.BuildTransferManager(storedTransfers); // Act - With a transfer status not in the above stored transfers TransferStatus[] statuses = new TransferStatus[] { new TransferStatus(TransferState.Stopping, true, false) }; @@ -218,7 +218,7 @@ public async Task GetTransfers_LocalCheckpointer() { CheckpointStoreOptions = TransferCheckpointStoreOptions.CreateLocalStore(test.DirectoryPath) }; - await using TransferManager manager = new TransferManager(options); + TransferManager manager = new TransferManager(options); // Act IList result = await manager.GetTransfersAsync().ToListAsync(); @@ -260,7 +260,7 @@ public async Task GetResumableTransfers_LocalCheckpointer() { CheckpointStoreOptions = TransferCheckpointStoreOptions.CreateLocalStore(test.DirectoryPath) }; - await using TransferManager manager = new TransferManager(options); + TransferManager manager = new TransferManager(options); // Act IList result = await manager.GetResumableTransfersAsync().ToListAsync(); @@ -299,7 +299,7 @@ public async Task GetResumableTransfers_IgnoresCompleted() { CheckpointStoreOptions = TransferCheckpointStoreOptions.CreateLocalStore(test.DirectoryPath) }; - await using TransferManager manager = new TransferManager(options); + TransferManager manager = new TransferManager(options); // Act IList result = await manager.GetResumableTransfersAsync().ToListAsync(); diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/PauseResumeTransferTestBase.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/PauseResumeTransferTestBase.cs index 87903793052e..d102f5e96baf 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/PauseResumeTransferTestBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/PauseResumeTransferTestBase.cs @@ -406,7 +406,7 @@ public async Task TryPauseTransferAsync_Id(TransferDirection transferType) ErrorMode = TransferErrorMode.ContinueOnFailure, ProvidersForResuming = new List() { provider }, }; - await using TransferManager transferManager = new TransferManager(options); + TransferManager transferManager = new TransferManager(options); TestProgressHandler progressHandler = new(); TransferOptions transferOptions = new TransferOptions { @@ -482,7 +482,7 @@ public async Task TryPauseTransferAsync_TransferOperation(TransferDirection tran } }; TestEventsRaised testEventsRaised = new TestEventsRaised(transferOptions); - await using TransferManager transferManager = new TransferManager(options); + TransferManager transferManager = new TransferManager(options); // Add long-running job to pause, if the job is not big enough // then the job might finish before we can pause it. @@ -520,7 +520,7 @@ public async Task TryPauseTransferAsync_TransferOperation(TransferDirection tran } [RecordedTest] - public async Task TryPauseTransferAsync_Error() + public void TryPauseTransferAsync_Error() { // Arrange using DisposingLocalDirectory checkpointerDirectory = DisposingLocalDirectory.GetTestDirectory(); @@ -529,7 +529,7 @@ public async Task TryPauseTransferAsync_Error() CheckpointStoreOptions = TransferCheckpointStoreOptions.CreateLocalStore(checkpointerDirectory.DirectoryPath), ErrorMode = TransferErrorMode.ContinueOnFailure }; - await using TransferManager transferManager = new TransferManager(options); + TransferManager transferManager = new TransferManager(options); // Act / Assert Assert.CatchAsync(async () => await transferManager.PauseTransferAsync("bad transfer Id")); @@ -566,7 +566,7 @@ public async Task TryPauseTransferAsync_AlreadyPaused(TransferDirection transfer } }; TestEventsRaised testEventsRaised = new TestEventsRaised(transferOptions); - await using TransferManager transferManager = new TransferManager(options); + TransferManager transferManager = new TransferManager(options); // Add long-running job to pause, if the job is not big enough // then the job might finish before we can pause it. @@ -632,7 +632,7 @@ public async Task PauseThenResumeTransferAsync(TransferDirection transferType) }; TransferOptions transferOptions = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(transferOptions); - await using TransferManager transferManager = new TransferManager(options); + TransferManager transferManager = new TransferManager(options); long size = DataMovementTestConstants.KB * 100; (StorageResource sResource, StorageResource dResource) = await CreateStorageResourcesAsync( @@ -720,7 +720,7 @@ public async Task ResumeTransferAsync(TransferDirection transferType) }; TransferOptions transferOptions = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(transferOptions); - await using TransferManager transferManager = new TransferManager(options); + TransferManager transferManager = new TransferManager(options); long size = DataMovementTestConstants.KB * 100; (StorageResource sResource, StorageResource dResource) = await CreateStorageResourcesAsync( @@ -784,7 +784,7 @@ public async Task ResumeTransferAsync_Options(TransferDirection transferType) ErrorMode = TransferErrorMode.ContinueOnFailure, ProvidersForResuming = new List() { provider }, }; - await using TransferManager transferManager = new TransferManager(options); + TransferManager transferManager = new TransferManager(options); Metadata metadata = DataProvider.BuildMetadata(); string contentLanguage = "en-US"; @@ -849,7 +849,7 @@ public async Task TryPauseTransferAsync_Id_Directory(TransferDirection transferT ErrorMode = TransferErrorMode.ContinueOnFailure, ProvidersForResuming = new List() { provider }, }; - await using TransferManager transferManager = new TransferManager(options); + TransferManager transferManager = new TransferManager(options); TransferOptions transferOptions = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(transferOptions); @@ -898,7 +898,7 @@ public async Task TryPauseTransferAsync_TransferOperation_Directory(TransferDire ErrorMode = TransferErrorMode.ContinueOnFailure, ProvidersForResuming = new List() { provider }, }; - await using TransferManager transferManager = new TransferManager(options); + TransferManager transferManager = new TransferManager(options); TransferOptions transferOptions = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(transferOptions); @@ -947,7 +947,7 @@ public async Task TryPauseTransferAsync_AlreadyPaused_Directory(TransferDirectio ErrorMode = TransferErrorMode.ContinueOnFailure, ProvidersForResuming = new List() { provider }, }; - await using TransferManager transferManager = new TransferManager(options); + TransferManager transferManager = new TransferManager(options); TransferOptions transferOptions = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(transferOptions); @@ -1007,7 +1007,7 @@ public async Task PauseThenResumeTransferAsync_Directory(TransferDirection trans ErrorMode = TransferErrorMode.ContinueOnFailure, ProvidersForResuming = new List() { provider }, }; - await using TransferManager transferManager = new TransferManager(options); + TransferManager transferManager = new TransferManager(options); TransferOptions transferOptions = new TransferOptions() { InitialTransferSize = DataMovementTestConstants.KB, @@ -1091,7 +1091,7 @@ public async Task ResumeTransferAsync_Directory(TransferDirection transferType) ErrorMode = TransferErrorMode.ContinueOnFailure, ProvidersForResuming = new List() { provider }, }; - await using TransferManager transferManager = new TransferManager(options); + TransferManager transferManager = new TransferManager(options); TransferOptions transferOptions = new TransferOptions() { InitialTransferSize = DataMovementTestConstants.KB, @@ -1182,7 +1182,7 @@ public async Task ResumeTransferAsync_Directory_Large( ErrorMode = TransferErrorMode.ContinueOnFailure, ProvidersForResuming = new List() { provider }, }; - await using TransferManager transferManager = new TransferManager(options); + TransferManager transferManager = new TransferManager(options); long size = DataMovementTestConstants.MB; (StorageResource sResource, StorageResource dResource) = await CreateStorageResourceContainersAsync( diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferCopyTestBase.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferCopyTestBase.cs index 00aa23bf6967..a0929795ddac 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferCopyTestBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferCopyTestBase.cs @@ -296,7 +296,7 @@ private async Task CopyRemoteObjectsAndVerify( List copyObjectInfo = new List(objectCount); // Initialize transfer manager - await using TransferManager transferManager = new TransferManager(transferManagerOptions); + TransferManager transferManager = new TransferManager(transferManagerOptions); // Upload set of VerifyCopyFromUriInfo Remote Objects to Copy for (int i = 0; i < objectCount; i++) @@ -582,7 +582,7 @@ public async Task SourceObjectToDestinationObject_Skip_Exists() StorageResourceItem destinationResource = GetDestinationStorageResourceItem(destinationClient); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager = new TransferManager(); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -642,7 +642,7 @@ public async Task SourceObjectToDestinationObject_Failure_Exists() objectLength: size); StorageResourceItem sourceResource = GetSourceStorageResourceItem(sourceClient); StorageResourceItem destinationResource = GetDestinationStorageResourceItem(destinationClient); - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager = new TransferManager(); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -678,24 +678,10 @@ await TestTransferWithTimeout.WaitForCompletionAsync( } #endregion - /// - /// For usage, please dispose of the TransferManager after use. - /// - /// Concurrency to set in TransferManagerOptions. - /// Valid TransferManager that needs to be disposed after use. - private TransferManager CreateTransferManager(int concurrency = default) - { - TransferManagerOptions managerOptions = new TransferManagerOptions() - { - MaximumConcurrency = concurrency, - }; - return new TransferManager(managerOptions); - } - private async Task CreateStartTransfer( TSourceContainerClient sourceContainer, TDestinationContainerClient destinationContainer, - TransferManager transferManager, + int concurrency, bool createFailedCondition = false, TransferOptions options = default, int size = DataMovementTestConstants.KB) @@ -719,6 +705,13 @@ private async Task CreateStartTransfer( StorageResourceItem sourceResource = GetSourceStorageResourceItem(sourceClient); StorageResourceItem destinationResource = GetDestinationStorageResourceItem(destinationClient); + // Create Transfer Manager with single threaded operation + TransferManagerOptions managerOptions = new TransferManagerOptions() + { + MaximumConcurrency = concurrency, + }; + TransferManager transferManager = new TransferManager(managerOptions); + // Start transfer and await for completion. return await transferManager.StartTransferAsync( sourceResource, @@ -732,7 +725,6 @@ public async Task StartTransfer_AwaitCompletion() // Arrange await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); - await using TransferManager transferManager = CreateTransferManager(concurrency: 1); TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); @@ -741,7 +733,7 @@ public async Task StartTransfer_AwaitCompletion() TransferOperation transfer = await CreateStartTransfer( source.Container, destination.Container, - transferManager, + concurrency: 1, options: options); // Act @@ -764,7 +756,6 @@ public async Task StartTransfer_AwaitCompletion_Failed() // Arrange await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); - await using TransferManager transferManager = CreateTransferManager(concurrency: 1); TransferOptions options = new TransferOptions() { @@ -776,7 +767,7 @@ public async Task StartTransfer_AwaitCompletion_Failed() TransferOperation transfer = await CreateStartTransfer( source.Container, destination.Container, - transferManager, + concurrency: 1, createFailedCondition: true, options: options); @@ -810,7 +801,6 @@ public async Task StartTransfer_AwaitCompletion_Skipped() // Arrange await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); - await using TransferManager transferManager = CreateTransferManager(concurrency: 1); // Create transfer options with Skipping available TransferOptions options = new TransferOptions() @@ -823,7 +813,7 @@ public async Task StartTransfer_AwaitCompletion_Skipped() TransferOperation transfer = await CreateStartTransfer( source.Container, destination.Container, - transferManager, + concurrency: 1, createFailedCondition: true, options: options); @@ -870,7 +860,7 @@ private async Task CopyRemoteObjects_VerifyProperties( MaximumTransferChunkSize = chunkSize, }; TestEventsRaised testEventsRaised = new TestEventsRaised(options); - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager = new TransferManager(); // Act - Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs index c54284d3bbac..ff34fabc93b9 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs @@ -255,7 +255,7 @@ private async Task CopyDirectoryAndVerifyAsync( }; // Initialize transferManager - await using TransferManager transferManager = new TransferManager(transferManagerOptions); + TransferManager transferManager = new TransferManager(transferManagerOptions); StorageResourceContainer sourceResource = GetSourceStorageResourceContainer(sourceContainer, sourcePrefix); @@ -383,7 +383,7 @@ public async Task DirectoryToDirectory_EmptyFolder() ErrorMode = TransferErrorMode.ContinueOnFailure, MaximumConcurrency = 1, }; - await using TransferManager transferManager = new TransferManager(managerOptions); + TransferManager transferManager = new TransferManager(managerOptions); TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); @@ -694,7 +694,7 @@ internal async Task CreateDirectoryTreeAsync( private async Task CreateStartTransfer( TSourceContainerClient sourceContainer, TDestinationContainerClient destinationContainer, - TransferManager transferManager, + int concurrency, bool createFailedCondition = false, TransferOptions options = default, int size = DataMovementTestConstants.KB) @@ -718,25 +718,18 @@ private async Task CreateStartTransfer( await CreateObjectInDestinationAsync(destinationContainer, size, fullDestPath); } - // Start transfer. - return await transferManager.StartTransferAsync( - sourceResource, - destinationResource, - options).ConfigureAwait(false); - } - - /// - /// For usage, please dispose of the TransferManager after use. - /// - /// Concurrency to set in TransferManagerOptions. - /// Valid TransferManager that needs to be disposed after use. - private TransferManager CreateTransferManager(int concurrency = default) - { + // Create Transfer Manager with single threaded operation TransferManagerOptions managerOptions = new TransferManagerOptions() { MaximumConcurrency = concurrency, }; - return new TransferManager(managerOptions); + TransferManager transferManager = new TransferManager(managerOptions); + + // Start transfer and await for completion. + return await transferManager.StartTransferAsync( + sourceResource, + destinationResource, + options).ConfigureAwait(false); } [RecordedTest] @@ -746,16 +739,13 @@ public async Task StartTransfer_AwaitCompletion() await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); - // Create Transfer Manager with single threaded operation - await using TransferManager transferManager = CreateTransferManager(concurrency:1); - // Create transfer to do a AwaitCompletion TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); TransferOperation transfer = await CreateStartTransfer( source.Container, destination.Container, - transferManager, + 1, options: options); // Act @@ -780,9 +770,6 @@ public async Task StartTransfer_AwaitCompletion_Failed() await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); - // Create Transfer Manager with single threaded operation - await using TransferManager transferManager = CreateTransferManager(concurrency: 1); - TransferOptions options = new TransferOptions() { CreationMode = StorageResourceCreationMode.FailIfExists @@ -793,7 +780,7 @@ public async Task StartTransfer_AwaitCompletion_Failed() TransferOperation transfer = await CreateStartTransfer( source.Container, destination.Container, - transferManager, + 1, createFailedCondition: true, options: options); @@ -820,9 +807,6 @@ public async Task StartTransfer_AwaitCompletion_Skipped() await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); - // Create Transfer Manager with single threaded operation - await using TransferManager transferManager = CreateTransferManager(concurrency: 1); - // Create transfer options with Skipping available TransferOptions options = new TransferOptions() { @@ -834,7 +818,7 @@ public async Task StartTransfer_AwaitCompletion_Skipped() TransferOperation transfer = await CreateStartTransfer( source.Container, destination.Container, - transferManager, + 1, createFailedCondition: true, options: options); @@ -861,9 +845,6 @@ public async Task StartTransfer_AwaitCompletion_Failed_SmallChunks() await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); - // Create Transfer Manager with single threaded operation - await using TransferManager transferManager = CreateTransferManager(concurrency: 1); - TransferOptions options = new TransferOptions() { CreationMode = StorageResourceCreationMode.FailIfExists, @@ -876,7 +857,7 @@ public async Task StartTransfer_AwaitCompletion_Failed_SmallChunks() TransferOperation transfer = await CreateStartTransfer( source.Container, destination.Container, - transferManager, + 1, createFailedCondition: true, options: options, size: DataMovementTestConstants.KB * 4); @@ -935,7 +916,7 @@ private async Task CopyRemoteObjects_VerifyProperties( // Create Transfer Manager TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager = new TransferManager(); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryDownloadTestBase.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryDownloadTestBase.cs index 501772a9413b..efca0d8b5797 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryDownloadTestBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryDownloadTestBase.cs @@ -242,7 +242,7 @@ public async Task DownloadDirectoryAsync_Empty() await SetupSourceDirectoryAsync(test.Container, sourceDirectoryName, new(), cancellationTokenSource.Token); // Initialize transferManager - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager = new TransferManager(); TransferOptions options = new TransferOptions(); TestEventsRaised testEventRaised = new TestEventsRaised(options); @@ -459,7 +459,7 @@ await SetupSourceDirectoryAsync( private async Task CreateStartTransfer( TContainerClient containerClient, string destinationFolder, - TransferManager transferManager, + int concurrency, TransferOptions options = default, int size = Constants.KB, CancellationToken cancellationToken = default) @@ -472,26 +472,19 @@ private async Task CreateStartTransfer( StorageResourceContainer sourceResource = GetStorageResourceContainer(containerClient, sourcePrefix); StorageResource destinationResource = LocalFilesStorageResourceProvider.FromDirectory(destinationFolder); - // Start transfer and await for completion. - return await transferManager.StartTransferAsync( - sourceResource, - destinationResource, - options).ConfigureAwait(false); - } - - /// - /// For usage, please dispose of the TransferManager after use. - /// - /// Concurrency to set in TransferManagerOptions. - /// Valid TransferManager that needs to be disposed after use. - private TransferManager CreateTransferManager(int concurrency = default) - { + // Create Transfer Manager with single threaded operation TransferManagerOptions managerOptions = new TransferManagerOptions() { MaximumConcurrency = concurrency, ErrorMode = TransferErrorMode.StopOnAnyFailure }; - return new TransferManager(managerOptions); + TransferManager transferManager = new TransferManager(managerOptions); + + // Start transfer and await for completion. + return await transferManager.StartTransferAsync( + sourceResource, + destinationResource, + options).ConfigureAwait(false); } [Test] @@ -503,16 +496,13 @@ public async Task StartTransfer_AwaitCompletion() string destinationFolder = CreateRandomDirectory(testDirectory.DirectoryPath); using CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); - // Create Transfer Manager with single threaded operation - await using TransferManager transferManager = CreateTransferManager(concurrency: 1); - // Create transfer to do a AwaitCompletion TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); TransferOperation transfer = await CreateStartTransfer( test.Container, destinationFolder, - transferManager, + 1, options: options, cancellationToken: cancellationTokenSource.Token); @@ -539,9 +529,6 @@ public async Task StartTransfer_AwaitCompletion_Failed() string destinationFolder = CreateRandomDirectory(testDirectory.DirectoryPath); using CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); - // Create Transfer Manager with single threaded operation - await using TransferManager transferManager = CreateTransferManager(concurrency: 1); - TransferOptions options = new TransferOptions() { CreationMode = StorageResourceCreationMode.FailIfExists @@ -555,7 +542,7 @@ public async Task StartTransfer_AwaitCompletion_Failed() TransferOperation transfer = await CreateStartTransfer( test.Container, destinationFolder, - transferManager, + 1, options: options, cancellationToken: cancellationTokenSource.Token); @@ -583,9 +570,6 @@ public async Task StartTransfer_AwaitCompletion_Skipped() string destinationFolder = CreateRandomDirectory(testDirectory.DirectoryPath); using CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); - // Create Transfer Manager with single threaded operation - await using TransferManager transferManager = CreateTransferManager(concurrency: 1); - // Create transfer options with Skipping available TransferOptions options = new TransferOptions() { @@ -600,7 +584,7 @@ public async Task StartTransfer_AwaitCompletion_Skipped() TransferOperation transfer = await CreateStartTransfer( test.Container, destinationFolder, - transferManager, + 1, options: options, cancellationToken: cancellationTokenSource.Token); @@ -628,9 +612,6 @@ public async Task StartTransfer_AwaitCompletion_Failed_SmallChunks() string destinationFolder = CreateRandomDirectory(testDirectory.DirectoryPath); using CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); - // Create Transfer Manager with single threaded operation - await using TransferManager transferManager = CreateTransferManager(concurrency: 1); - TransferOptions options = new TransferOptions() { CreationMode = StorageResourceCreationMode.FailIfExists, @@ -646,7 +627,7 @@ public async Task StartTransfer_AwaitCompletion_Failed_SmallChunks() TransferOperation transfer = await CreateStartTransfer( test.Container, destinationFolder, - transferManager, + 1, options: options, size: Constants.KB * 4, cancellationToken: cancellationTokenSource.Token); diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDownloadTestBase.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDownloadTestBase.cs index 41f6a6b57654..593ab5cfcfce 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDownloadTestBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDownloadTestBase.cs @@ -114,7 +114,7 @@ protected string GetNewObjectName() private async Task CreateStartTransfer( TContainerClient containerClient, string localDirectoryPath, - TransferManager transferManager, + int concurrency, bool createFailedCondition = false, TransferOptions options = default, int size = DataMovementTestConstants.KB) @@ -141,6 +141,13 @@ private async Task CreateStartTransfer( StorageResourceItem sourceResource = GetStorageResourceItem(TObjectClient); StorageResource destinationResource = LocalFilesStorageResourceProvider.FromFile(destFile); + // Create Transfer Manager with single threaded operation + TransferManagerOptions managerOptions = new TransferManagerOptions() + { + MaximumConcurrency = concurrency, + }; + TransferManager transferManager = new TransferManager(managerOptions); + // Start transfer and await for completion. return await transferManager.StartTransferAsync( sourceResource, @@ -148,21 +155,6 @@ private async Task CreateStartTransfer( options).ConfigureAwait(false); } - /// - /// For usage, please dispose of the TransferManager after use. - /// - /// Concurrency to set in TransferManagerOptions. - /// Valid TransferManager that needs to be disposed after use. - private TransferManager CreateTransferManager(int concurrency = default) - { - TransferManagerOptions managerOptions = new TransferManagerOptions() - { - MaximumConcurrency = concurrency, - ErrorMode = TransferErrorMode.StopOnAnyFailure - }; - return new TransferManager(managerOptions); - } - [RecordedTest] public async Task StartTransfer_AwaitCompletion() { @@ -170,16 +162,13 @@ public async Task StartTransfer_AwaitCompletion() await using IDisposingContainer test = await GetDisposingContainerAsync(); using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); - // Create Transfer Manager with single threaded operation - await using TransferManager transferManager = CreateTransferManager(concurrency: 1); - // Create transfer to do a AwaitCompletion TransferOptions options = new TransferOptions(); TestEventsRaised failureTransferHolder = new TestEventsRaised(options); TransferOperation transfer = await CreateStartTransfer( containerClient: test.Container, localDirectoryPath: testDirectory.DirectoryPath, - transferManager: transferManager, + concurrency: 1, options: options); // Act @@ -203,9 +192,6 @@ public async Task StartTransfer_AwaitCompletion_Failed() await using IDisposingContainer test = await GetDisposingContainerAsync(); using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); - // Create Transfer Manager with single threaded operation - await using TransferManager transferManager = CreateTransferManager(concurrency: 1); - TransferOptions options = new TransferOptions() { CreationMode = StorageResourceCreationMode.FailIfExists @@ -216,7 +202,7 @@ public async Task StartTransfer_AwaitCompletion_Failed() TransferOperation transfer = await CreateStartTransfer( containerClient: test.Container, localDirectoryPath: testDirectory.DirectoryPath, - transferManager: transferManager, + concurrency: 1, createFailedCondition: true, options: options); @@ -244,9 +230,6 @@ public async Task StartTransfer_AwaitCompletion_Skipped() await using IDisposingContainer test = await GetDisposingContainerAsync(); using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); - // Create Transfer Manager with single threaded operation - await using TransferManager transferManager = CreateTransferManager(concurrency: 1); - // Create transfer options with Skipping available TransferOptions options = new TransferOptions() { @@ -258,7 +241,7 @@ public async Task StartTransfer_AwaitCompletion_Skipped() TransferOperation transfer = await CreateStartTransfer( containerClient: test.Container, localDirectoryPath: testDirectory.DirectoryPath, - transferManager: transferManager, + concurrency: 1, createFailedCondition: true, options: options); @@ -303,7 +286,7 @@ public VerifyDownloadObjectContentInfo( /// /// Upload and verify the contents of the object /// - /// By default in this function an event argument will be added to the options event handler + /// By default in this function an event arguement will be added to the options event handler /// to detect when the upload has finished. /// /// @@ -346,7 +329,7 @@ private async Task DownloadObjectsAndVerify( List downloadedObjectInfo = new List(objectCount); // Initialize TransferManager - await using TransferManager transferManager = new TransferManager(transferManagerOptions); + TransferManager transferManager = new TransferManager(transferManagerOptions); // Upload set of VerifyDownloadObjectContentInfo objects to download for (int i = 0; i < objectCount; i++) { @@ -515,7 +498,7 @@ public async Task RemoteObjectToLocal_Skip_Exists() return Task.CompletedTask; }; TestEventsRaised testEventsRaised = new(options); - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager = new TransferManager(); StorageResource destinationResource = LocalFilesStorageResourceProvider.FromFile(destFile); @@ -568,7 +551,7 @@ public async Task RemoteObjectToLocal_Failure_Exists() StorageResourceItem sourceResource = GetStorageResourceItem(sourceClient); StorageResource destinationResource = LocalFilesStorageResourceProvider.FromFile(destFile); - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager = new TransferManager(); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferUploadDirectoryTestBase.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferUploadDirectoryTestBase.cs index dd4ef9d3d1ea..ddaf7ceeaae9 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferUploadDirectoryTestBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferUploadDirectoryTestBase.cs @@ -132,7 +132,7 @@ private async Task UploadDirectoryAndVerifyAsync( string sourceLocalDirectoryPath, TContainerClient destinationContainer, int expectedTransfers, - TransferManager transferManager, + TransferManagerOptions transferManagerOptions = default, TransferOptions options = default, CancellationToken cancellationToken = default) { @@ -140,12 +140,17 @@ private async Task UploadDirectoryAndVerifyAsync( options ??= new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); + transferManagerOptions ??= new TransferManagerOptions() + { + ErrorMode = TransferErrorMode.ContinueOnFailure + }; + StorageResourceContainer sourceResource = LocalFilesStorageResourceProvider.FromDirectory(sourceLocalDirectoryPath); StorageResourceContainer destinationResource = GetStorageResourceContainer(destinationContainer); await new TransferValidator() { - TransferManager = transferManager + TransferManager = new(transferManagerOptions) }.TransferAndVerifyAsync( sourceResource, destinationResource, @@ -155,20 +160,6 @@ private async Task UploadDirectoryAndVerifyAsync( options, cancellationToken); } - - /// - /// For usage, please dispose of the TransferManager after use. - /// - /// Concurrency to set in TransferManagerOptions. - /// Valid TransferManager that needs to be disposed after use. - private TransferManager CreateTransferManager() - { - TransferManagerOptions transferManagerOptions = new TransferManagerOptions() - { - ErrorMode = TransferErrorMode.ContinueOnFailure - }; - return new TransferManager(transferManagerOptions); - } #endregion [RecordedTest] @@ -184,7 +175,6 @@ public async Task Upload(long objectSize, int waitTimeInSec) // Arrange using DisposingLocalDirectory disposingLocalDirectory = DisposingLocalDirectory.GetTestDirectory(); await using IDisposingContainer test = await GetDisposingContainerAsync(); - await using TransferManager transferManager = CreateTransferManager(); List files = new() { @@ -202,7 +192,6 @@ await UploadDirectoryAndVerifyAsync( disposingLocalDirectory.DirectoryPath, test.Container, expectedTransfers: files.Count, - transferManager: transferManager, cancellationToken: cts.Token); } } @@ -313,30 +302,33 @@ await SetupDirectoryAsync( CreationMode = StorageResourceCreationMode.SkipIfExists }; TestEventsRaised testEventsRaised = new TestEventsRaised(options); - await using TransferManager transferManager = CreateTransferManager(); + TransferManagerOptions transferManagerOptions = new() { - StorageResourceContainer sourceResource = LocalFilesStorageResourceProvider.FromDirectory(disposingLocalDirectory.DirectoryPath); - StorageResourceContainer destinationResource = GetStorageResourceContainer(test.Container); - TransferOperation transfer = await transferManager.StartTransferAsync(sourceResource, destinationResource, options, cts.Token); - await TestTransferWithTimeout.WaitForCompletionAsync( - transfer, - testEventsRaised, - cts.Token); + ErrorMode = TransferErrorMode.ContinueOnFailure + }; - // check if expected files exist, but not necessarily for contents - await testEventsRaised.AssertContainerCompletedWithSkippedCheck(preexistingFileCount); + StorageResourceContainer sourceResource = LocalFilesStorageResourceProvider.FromDirectory(disposingLocalDirectory.DirectoryPath); + StorageResourceContainer destinationResource = GetStorageResourceContainer(test.Container); + TransferOperation transfer = await new TransferManager(transferManagerOptions) + .StartTransferAsync(sourceResource, destinationResource, options, cts.Token); + await TestTransferWithTimeout.WaitForCompletionAsync( + transfer, + testEventsRaised, + cts.Token); - // Verify all files exist, meaning files without conflict were transferred. - List localFiles = (await TransferValidator.GetLocalFileLister(disposingLocalDirectory.DirectoryPath) - .Invoke(cts.Token)) - .Select(item => item.RelativePath) - .ToList(); - List destinationObjects = (await GetStorageResourceLister(test.Container) - .Invoke(cts.Token)) - .Select(item => item.RelativePath) - .ToList(); - Assert.That(localFiles, Is.EquivalentTo(destinationObjects)); - } + // check if expected files exist, but not necessarily for contents + await testEventsRaised.AssertContainerCompletedWithSkippedCheck(preexistingFileCount); + + // Verify all files exist, meaning files without conflict were transferred. + List localFiles = (await TransferValidator.GetLocalFileLister(disposingLocalDirectory.DirectoryPath) + .Invoke(cts.Token)) + .Select(item => item.RelativePath) + .ToList(); + List destinationObjects = (await GetStorageResourceLister(test.Container) + .Invoke(cts.Token)) + .Select(item => item.RelativePath) + .ToList(); + Assert.That(localFiles, Is.EquivalentTo(destinationObjects)); } } @@ -369,16 +361,12 @@ await SetupDirectoryAsync( disposingLocalDirectory.DirectoryPath, files.Select(path => (path, DefaultObjectSize)).ToList(), cts.Token); - await using TransferManager transferManager = CreateTransferManager(); - { - await UploadDirectoryAndVerifyAsync( + await UploadDirectoryAndVerifyAsync( disposingLocalDirectory.DirectoryPath, test.Container, expectedTransfers: files.Count, - transferManager: transferManager, options: options, cancellationToken: cts.Token); - } } } @@ -410,12 +398,10 @@ await SetupDirectoryAsync( disposingLocalDirectory.DirectoryPath, files.Select(path => (path, objectSize)).ToList(), cts.Token); - await using TransferManager transferManager = CreateTransferManager(); await UploadDirectoryAndVerifyAsync( disposingLocalDirectory.DirectoryPath, test.Container, expectedTransfers: files.Count, - transferManager: transferManager, cancellationToken: cts.Token); } } @@ -444,15 +430,13 @@ void BuildFolders(string path, int depth) } BuildFolders(disposingLocalDirectory.DirectoryPath, folderDepth); - await using (TransferManager transferManager = CreateTransferManager()) using (CancellationTokenSource cts = TestHelper.GetTimeoutTokenSource(waitTimeInSec)) { await UploadDirectoryAndVerifyAsync( - disposingLocalDirectory.DirectoryPath, - test.Container, - expectedTransfers: 0, - transferManager: transferManager, - cancellationToken: cts.Token); + disposingLocalDirectory.DirectoryPath, + test.Container, + expectedTransfers: 0, + cancellationToken: cts.Token); } } @@ -487,13 +471,11 @@ await SetupDirectoryAsync( disposingLocalDirectory.DirectoryPath, files.Select(path => (path, DefaultObjectSize)).ToList(), cts.Token); - await using TransferManager transferManager = CreateTransferManager(); await UploadDirectoryAndVerifyAsync( disposingLocalDirectory.DirectoryPath, test.Container, - expectedTransfers: 1 << folderDepth, - transferManager: transferManager, - cancellationToken: cts.Token); + expectedTransfers: 1 << folderDepth, + cancellationToken: cts.Token); } } @@ -518,13 +500,10 @@ await SetupDirectoryAsync( disposingLocalDirectory.DirectoryPath, files.Select(path => (path, DefaultObjectSize)).ToList(), cts.Token); - - await using TransferManager transferManager = CreateTransferManager(); await UploadDirectoryAndVerifyAsync( disposingLocalDirectory.DirectoryPath, test.Container, expectedTransfers: 1, - transferManager: transferManager, cancellationToken: cts.Token); } } @@ -549,7 +528,6 @@ public async Task Upload_SpecialChars(string prefix) string.Join("/", "space folder", "space file"), ]; - await using TransferManager transferManager = CreateTransferManager(); using (CancellationTokenSource cts = TestHelper.GetTimeoutTokenSource(30)) { await SetupDirectoryAsync( @@ -560,7 +538,6 @@ await UploadDirectoryAndVerifyAsync( disposingLocalDirectory.DirectoryPath, test.Container, expectedTransfers: files.Count, - transferManager: transferManager, cancellationToken: cts.Token); } } @@ -574,22 +551,18 @@ public async Task Upload_TrailingSlash() List files = [ "file1", "file2", "dir1/file1" ]; using CancellationTokenSource cancellationTokenSource = TestHelper.GetTimeoutTokenSource(30); - { - await SetupDirectoryAsync( - disposingLocalDirectory.DirectoryPath, - files.Select(path => (path, (long)Constants.KB)).ToList(), - cancellationTokenSource.Token); - - // Intentionally append trailing slash - string sourcePath = disposingLocalDirectory.DirectoryPath + Path.DirectorySeparatorChar; - await using TransferManager transferManager = CreateTransferManager(); - await UploadDirectoryAndVerifyAsync( - sourcePath, - test.Container, - expectedTransfers: files.Count, - transferManager: transferManager, - cancellationToken: cancellationTokenSource.Token); - } + await SetupDirectoryAsync( + disposingLocalDirectory.DirectoryPath, + files.Select(path => (path, (long)Constants.KB)).ToList(), + cancellationTokenSource.Token); + + // Intentionally append trailing slash + string sourcePath = disposingLocalDirectory.DirectoryPath + Path.DirectorySeparatorChar; + await UploadDirectoryAndVerifyAsync( + sourcePath, + test.Container, + expectedTransfers: files.Count, + cancellationToken: cancellationTokenSource.Token); } } } diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferUploadTestBase.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferUploadTestBase.cs index e46bae110487..2439b023af04 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferUploadTestBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferUploadTestBase.cs @@ -120,7 +120,7 @@ protected string GetNewObjectName() private async Task CreateStartTransfer( TContainerClient containerClient, string localDirectoryPath, - TransferManager transferManager, + int concurrency, bool createFailedCondition = false, TransferOptions options = default, int size = DataMovementTestConstants.KB) @@ -147,6 +147,13 @@ private async Task CreateStartTransfer( } StorageResource sourceResource = LocalFilesStorageResourceProvider.FromFile(localSourceFile); + // Create Transfer Manager with single threaded operation + TransferManagerOptions managerOptions = new TransferManagerOptions() + { + MaximumConcurrency = concurrency, + }; + TransferManager transferManager = new TransferManager(managerOptions); + // Start transfer and await for completion. return await transferManager.StartTransferAsync( sourceResource, @@ -154,20 +161,6 @@ private async Task CreateStartTransfer( options).ConfigureAwait(false); } - /// - /// For usage, please dispose of the TransferManager after use. - /// - /// Concurrency to set in TransferManagerOptions. - /// Valid TransferManager that needs to be disposed after use. - private TransferManager CreateTransferManager(int concurrency = default) - { - TransferManagerOptions managerOptions = new TransferManagerOptions() - { - MaximumConcurrency = concurrency, - }; - return new TransferManager(managerOptions); - } - [RecordedTest] public async Task StartTransfer_AwaitCompletion() { @@ -175,9 +168,6 @@ public async Task StartTransfer_AwaitCompletion() using DisposingLocalDirectory disposingLocalDirectory = DisposingLocalDirectory.GetTestDirectory(); await using IDisposingContainer test = await GetDisposingContainerAsync(); - // Create Transfer Manager with single threaded operation - await using TransferManager transferManager = CreateTransferManager(concurrency: 1); - TransferOptions options = new TransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); @@ -185,7 +175,7 @@ public async Task StartTransfer_AwaitCompletion() TransferOperation transfer = await CreateStartTransfer( containerClient: test.Container, localDirectoryPath: disposingLocalDirectory.DirectoryPath, - transferManager: transferManager, + 1, options: options); // Act @@ -209,9 +199,6 @@ public async Task StartTransfer_AwaitCompletion_Failed() using DisposingLocalDirectory disposingLocalDirectory = DisposingLocalDirectory.GetTestDirectory(); await using IDisposingContainer test = await GetDisposingContainerAsync(); - // Create Transfer Manager with single threaded operation - await using TransferManager transferManager = CreateTransferManager(concurrency: 1); - TransferOptions options = new TransferOptions() { CreationMode = StorageResourceCreationMode.FailIfExists @@ -222,7 +209,7 @@ public async Task StartTransfer_AwaitCompletion_Failed() TransferOperation transfer = await CreateStartTransfer( containerClient: test.Container, localDirectoryPath: disposingLocalDirectory.DirectoryPath, - transferManager: transferManager, + concurrency: 1, createFailedCondition: true, options: options); @@ -258,9 +245,6 @@ public async Task StartTransfer_AwaitCompletion_Skipped() using DisposingLocalDirectory disposingLocalDirectory = DisposingLocalDirectory.GetTestDirectory(); await using IDisposingContainer test = await GetDisposingContainerAsync(); - // Create Transfer Manager with single threaded operation - await using TransferManager transferManager = CreateTransferManager(concurrency: 1); - // Create transfer options with Skipping available TransferOptions options = new TransferOptions() { @@ -272,7 +256,7 @@ public async Task StartTransfer_AwaitCompletion_Skipped() TransferOperation transfer = await CreateStartTransfer( containerClient: test.Container, localDirectoryPath: disposingLocalDirectory.DirectoryPath, - transferManager: transferManager, + concurrency: 1, createFailedCondition: true, options: options); @@ -359,7 +343,7 @@ private async Task UploadResourceAndVerify( List uploadedObjectInfo = new List(objectCount); // Initialize TransferManager - await using TransferManager transferManager = new TransferManager(transferManagerOptions); + TransferManager transferManager = new TransferManager(transferManagerOptions); // Set up file to upload for (int i = 0; i < objectCount; i++) @@ -568,7 +552,7 @@ public async Task LocalToRemoteObject_Skip_Exists() StorageResourceItem destinationResource = GetStorageResourceItem(objectClient); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager = new TransferManager(); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( @@ -625,7 +609,7 @@ public async Task LocalToRemoteObject_Failure_Exists() TestEventsRaised testEventRaised = new TestEventsRaised(options); StorageResource sourceResource = LocalFilesStorageResourceProvider.FromFile(newSourceFile); StorageResourceItem destinationResource = GetStorageResourceItem(objectClient); - await using TransferManager transferManager = new TransferManager(); + TransferManager transferManager = new TransferManager(); // Start transfer and await for completion. TransferOperation transfer = await transferManager.StartTransferAsync( diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/TransferOperationTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/TransferOperationTests.cs index 4335f2d30332..3271e998b924 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/TransferOperationTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/TransferOperationTests.cs @@ -170,11 +170,10 @@ public async Task TryPauseAsync_CancellationToken() // Arrange string transferId = GetNewTransferId(); - await using TransferManager transferManager = new TransferManager(); TransferOperation transfer = new TransferOperation( id: transferId, status: InProgressStatus); - transfer.TransferManager = transferManager; + transfer.TransferManager = new TransferManager(); CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(1)); try From 6d3ab6893cf566d25afec36902d7ec54aa7f0a04 Mon Sep 17 00:00:00 2001 From: Amanda Nguyen Date: Wed, 8 Oct 2025 16:48:24 -0700 Subject: [PATCH 8/8] Prevent state change after transfer has been completed --- .../src/TransferInternalState.cs | 45 ++++++++++--------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/sdk/storage/Azure.Storage.DataMovement/src/TransferInternalState.cs b/sdk/storage/Azure.Storage.DataMovement/src/TransferInternalState.cs index c2d898b32faa..3218d6f49d93 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/TransferInternalState.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/TransferInternalState.cs @@ -91,29 +91,34 @@ public TransferStatus GetTransferStatus() /// Returns whether or not the status has been changed from its original state. public bool SetTransferState(TransferState state) { - Argument.AssertNotNull(TransferManager, nameof(TransferManager)); - if (_status.SetTransferStateChange(state)) + // Only allow state changes if not already in a Completed/Paused state. + if (TransferState.Completed != _status.State && + TransferState.Paused != _status.State) { - if (TransferState.Completed == _status.State || - TransferState.Paused == _status.State) + Argument.AssertNotNull(TransferManager, nameof(TransferManager)); + if (_status.SetTransferStateChange(state)) { - DataMovementEventSource.Singleton.TransferCompleted(Id, _status); - // If the _completionSource has been cancelled or the exception - // has been set, we don't need to check if TrySetResult returns false - // because it's acceptable to cancel or have an error occur before then. - - CompletionSource.TrySetResult(_status); - - // Tell the transfer manager to clean up the completed/paused job. - TransferManager.TryRemoveTransfer(_id); - - // Remove Transfer Manager reference after no longer needed. - TransferManager = null; - - // Once we reach a Completed/Paused, Dispose the CancellationTokenSource to release resources (since it is no longer needed). - DisposeCancellationTokenSource(); + if (TransferState.Completed == _status.State || + TransferState.Paused == _status.State) + { + DataMovementEventSource.Singleton.TransferCompleted(Id, _status); + // If the _completionSource has been cancelled or the exception + // has been set, we don't need to check if TrySetResult returns false + // because it's acceptable to cancel or have an error occur before then. + + CompletionSource.TrySetResult(_status); + + // Tell the transfer manager to clean up the completed/paused job. + TransferManager.TryRemoveTransfer(_id); + + // Remove Transfer Manager reference after no longer needed. + TransferManager = null; + + // Once we reach a Completed/Paused, Dispose the CancellationTokenSource to release resources (since it is no longer needed). + DisposeCancellationTokenSource(); + } + return true; } - return true; } return false; }