diff --git a/sdk/storage/Azure.Storage.Common/src/stress/Shared/Metrics.cs b/sdk/storage/Azure.Storage.Common/src/stress/Shared/Metrics.cs index 01ad1eb637c9..cf66483fe54c 100644 --- a/sdk/storage/Azure.Storage.Common/src/stress/Shared/Metrics.cs +++ b/sdk/storage/Azure.Storage.Common/src/stress/Shared/Metrics.cs @@ -45,14 +45,29 @@ public class Metrics /// This is the metric name used to collect how many objects (e.g. local file, blob, share file) were transferred. /// /// - public const string ObjectsTransferred = "ObjectsTransferred"; + public const string ItemTransferCompleted = "ItemTransferCompleted"; + + /// + /// This is the metric name used to collect each transfer status that occurred. + /// + public const string TransferStatusChanged = "TransferStatusChanged"; + + /// + /// This is the metric name used to keep track of the failed transfer items. + /// + public const string TransferFailedItem = "TransferFailedItem"; + + /// + /// This is the metric name used to keep track of the skipped transfer items + /// + public const string ItemTransferSkipped = "ItemTransferSkipped"; /// /// This is the metric name used to collect metrics on how many times /// the individual transfer part was restarted or retried. /// /// - public const string TransferRetriedRestarted = "TransferRestarted"; + public const string TransferRestarted = "TransferRestarted"; /// /// This is the metric name used to collect metrics on the total number of bytes that have been diff --git a/sdk/storage/Azure.Storage.Common/src/stress/Shared/TestScenarioBase.cs b/sdk/storage/Azure.Storage.Common/src/stress/Shared/TestScenarioBase.cs index 57a9a48fa078..be01c8b75172 100644 --- a/sdk/storage/Azure.Storage.Common/src/stress/Shared/TestScenarioBase.cs +++ b/sdk/storage/Azure.Storage.Common/src/stress/Shared/TestScenarioBase.cs @@ -21,7 +21,7 @@ public abstract class TestScenarioBase /// /// Initializes a new Test instance. /// - ///\ + /// /// The to use to send metrics to Application Insights. /// Test Run Id to differ between test runs. /// diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/generatedValues.yaml b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/generatedValues.yaml index 24c98afdb264..ea278c5eecfe 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/generatedValues.yaml +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/generatedValues.yaml @@ -1,92 +1,92 @@ scenarios: - image: Dockerfile - imageBuildDir: ../../../.. Scenario: uploadsingleblockblob testScenario: uploadsingleblockblob + imageBuildDir: ../../../.. imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye - image: Dockerfile + Scenario: uploaddirectoryblockblob + testScenario: uploaddirectoryblockblob imageBuildDir: ../../../.. - Scenario: uploaddirectoryblockBlob - testScenario: uploaddirectoryblockBlob imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye - image: Dockerfile - imageBuildDir: ../../../.. Scenario: downloadsingleblockblob testScenario: downloadsingleblockblob + imageBuildDir: ../../../.. imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye - image: Dockerfile - imageBuildDir: ../../../.. Scenario: downloaddirectoryblockblob testScenario: downloaddirectoryblockblob + imageBuildDir: ../../../.. imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye - image: Dockerfile - imageBuildDir: ../../../.. Scenario: copysingleblockblob testScenario: copysingleblockblob + imageBuildDir: ../../../.. imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye - image: Dockerfile - imageBuildDir: ../../../.. Scenario: copydirectoryblockblob testScenario: copydirectoryblockblob + imageBuildDir: ../../../.. imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye - image: Dockerfile - imageBuildDir: ../../../.. Scenario: uploadsingleappendblob testScenario: uploadsingleappendblob + imageBuildDir: ../../../.. imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye - image: Dockerfile - imageBuildDir: ../../../.. Scenario: uploaddirectoryappendblob testScenario: uploaddirectoryappendblob + imageBuildDir: ../../../.. imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye - image: Dockerfile - imageBuildDir: ../../../.. Scenario: downloadsingleappendblob testScenario: downloadsingleappendblob + imageBuildDir: ../../../.. imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye - image: Dockerfile - imageBuildDir: ../../../.. Scenario: downloaddirectoryappendblob testScenario: downloaddirectoryappendblob + imageBuildDir: ../../../.. imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye - image: Dockerfile - imageBuildDir: ../../../.. Scenario: copysingleappendblob testScenario: copysingleappendblob + imageBuildDir: ../../../.. imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye - image: Dockerfile - imageBuildDir: ../../../.. Scenario: copydirectoryappendblob testScenario: copydirectoryappendblob + imageBuildDir: ../../../.. imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye - image: Dockerfile - imageBuildDir: ../../../.. Scenario: uploadsinglepageblob testScenario: uploadsinglepageblob + imageBuildDir: ../../../.. imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye - image: Dockerfile - imageBuildDir: ../../../.. Scenario: uploaddirectorypageblob testScenario: uploaddirectorypageblob + imageBuildDir: ../../../.. imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye - image: Dockerfile - imageBuildDir: ../../../.. Scenario: downloadsinglepageblob testScenario: downloadsinglepageblob + imageBuildDir: ../../../.. imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye - image: Dockerfile - imageBuildDir: ../../../.. Scenario: downloaddirectorypageblob testScenario: downloaddirectorypageblob + imageBuildDir: ../../../.. imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye - image: Dockerfile - imageBuildDir: ../../../.. Scenario: copysinglepageblob testScenario: copysinglepageblob + imageBuildDir: ../../../.. imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye - image: Dockerfile - imageBuildDir: ../../../.. Scenario: copydirectorypageblob testScenario: copydirectorypageblob + imageBuildDir: ../../../.. imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Azure.Storage.DataMovement.Blobs.Stress.csproj b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Azure.Storage.DataMovement.Blobs.Stress.csproj index 3d3602d85b43..9b66aefc6836 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Azure.Storage.DataMovement.Blobs.Stress.csproj +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Azure.Storage.DataMovement.Blobs.Stress.csproj @@ -21,7 +21,9 @@ - + + BaseBlobs + @@ -37,6 +39,8 @@ + + @@ -46,11 +50,13 @@ - + + + @@ -58,3 +64,4 @@ + diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Configurations/BlockBlobConfiguration.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Configurations/BlockBlobConfiguration.cs index 7a2278e052af..babccb972c90 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Configurations/BlockBlobConfiguration.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Configurations/BlockBlobConfiguration.cs @@ -1,7 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +extern alias BaseBlobs; -using Azure.Storage.Blobs; +using BaseBlobs::Azure.Storage.Blobs; namespace Azure.Storage.DataMovement.Blobs.Stress { diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Infrastructure/BlobTestSetupHelper.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Infrastructure/BlobTestSetupHelper.cs index 69d812380e10..188530f4ddd6 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Infrastructure/BlobTestSetupHelper.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Infrastructure/BlobTestSetupHelper.cs @@ -1,15 +1,17 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +extern alias BaseBlobs; + using System; using System.IO; using System.Threading; using System.Threading.Tasks; using Azure.Storage.Shared; -using Azure.Storage.Blobs; -using Azure.Storage.Blobs.Specialized; +using BaseBlobs::Azure.Storage.Blobs; +using BaseBlobs::Azure.Storage.Blobs.Specialized; using NUnit.Framework; -using Azure.Storage.Blobs.Models; +using BaseBlobs::Azure.Storage.Blobs.Models; namespace Azure.Storage.DataMovement.Blobs.Stress { diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/BlobScenarioBase.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/BlobScenarioBase.cs index 6b6b98f89de1..f3a0a303a88c 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/BlobScenarioBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/BlobScenarioBase.cs @@ -1,17 +1,20 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +extern alias BaseBlobs; + using System; using Azure.Core; using Azure.Storage.Stress; -using Azure.Storage.Blobs; +using BaseBlobs::Azure.Storage.Blobs; using System.Threading.Tasks; using System.IO; using System.Threading; +using Azure.Storage.DataMovement.Stress; namespace Azure.Storage.DataMovement.Blobs.Stress { - public abstract class BlobScenarioBase : TestScenarioBase + public abstract class BlobScenarioBase : DataMovementScenarioBase { protected internal readonly Uri _destinationBlobUri; protected internal int _blobSize; @@ -19,8 +22,6 @@ public abstract class BlobScenarioBase : TestScenarioBase protected internal BlobsStorageResourceProvider _blobsStorageResourceProvider; protected internal LocalFilesStorageResourceProvider _localFilesStorageResourceProvider; protected internal BlobServiceClient _blobServiceClient; - protected internal readonly TransferManagerOptions _transferManagerOptions; - protected internal readonly DataTransferOptions _dataTransferOptions; public BlobScenarioBase( Uri blobUri, @@ -30,12 +31,10 @@ public BlobScenarioBase( TokenCredential tokenCredential, Metrics metrics, string testRunId) - : base(metrics, testRunId) + : base(transferManagerOptions, dataTransferOptions, metrics, testRunId) { _destinationBlobUri = blobUri; _blobSize = blobSize != default ? blobSize.Value : DataMovementBlobStressConstants.DefaultObjectSize; - _transferManagerOptions = transferManagerOptions; - _dataTransferOptions = dataTransferOptions; _tokenCredential = tokenCredential; _blobsStorageResourceProvider = new BlobsStorageResourceProvider(tokenCredential); _localFilesStorageResourceProvider = new LocalFilesStorageResourceProvider(); diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyAppendBlobDirectoryScenario.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyAppendBlobDirectoryScenario.cs index ce04f9559019..bd6e8256802c 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyAppendBlobDirectoryScenario.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyAppendBlobDirectoryScenario.cs @@ -1,11 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +extern alias BaseBlobs; using System; using System.Threading; using System.Threading.Tasks; using Azure.Core; -using Azure.Storage.Blobs.Models; +using BaseBlobs::Azure.Storage.Blobs.Models; using Azure.Storage.Stress; namespace Azure.Storage.DataMovement.Blobs.Stress diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyAppendBlobSingleScenario.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyAppendBlobSingleScenario.cs index fae8005dbace..226a0929cff4 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyAppendBlobSingleScenario.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyAppendBlobSingleScenario.cs @@ -1,11 +1,13 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +extern alias BaseBlobs; + using System; using System.Threading; using System.Threading.Tasks; using Azure.Core; -using Azure.Storage.Blobs.Models; +using BaseBlobs::Azure.Storage.Blobs.Models; using Azure.Storage.Stress; namespace Azure.Storage.DataMovement.Blobs.Stress diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyBlobDirectoryScenarioBase.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyBlobDirectoryScenarioBase.cs index 893d5fc0a566..f96bf6df8bc0 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyBlobDirectoryScenarioBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyBlobDirectoryScenarioBase.cs @@ -1,13 +1,16 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +extern alias BaseBlobs; + using System; using System.Threading; using System.Threading.Tasks; using Azure.Core; -using Azure.Storage.Blobs; -using Azure.Storage.Blobs.Models; +using BaseBlobs::Azure.Storage.Blobs; +using BaseBlobs::Azure.Storage.Blobs.Models; using Azure.Storage.DataMovement.Tests; +using Azure.Storage.DataMovement.Blobs.Tests; using Azure.Storage.Stress; namespace Azure.Storage.DataMovement.Blobs.Stress @@ -36,46 +39,90 @@ public CopyBlobDirectoryScenarioBase( public async Task RunTestInternalAsync(BlobType blobType, CancellationToken cancellationToken) { - string pathPrefix = TestSetupHelper.Randomize("dir"); - string sourceContainerName = TestSetupHelper.Randomize("container"); - BlobContainerClient sourceContainerClient = _sourceServiceClient.GetBlobContainerClient(sourceContainerName); - await sourceContainerClient.CreateIfNotExistsAsync(); - await BlobTestSetupHelper.CreateBlobsInDirectoryAsync( - sourceContainerClient, - blobType, - pathPrefix, - _blobCount, - _blobSize, - cancellationToken); + while (!cancellationToken.IsCancellationRequested) + { + string sourceContainerName = TestSetupHelper.Randomize("container"); + DisposingBlobContainer sourceDisposingContainer = new(_sourceServiceClient.GetBlobContainerClient(sourceContainerName)); + string destinationContainerName = TestSetupHelper.Randomize("container"); + DisposingBlobContainer destinationDisposingContainer = new(_blobServiceClient.GetBlobContainerClient(destinationContainerName)); + try + { + string pathPrefix = TestSetupHelper.Randomize("dir"); + BlobContainerClient sourceContainerClient = sourceDisposingContainer.Container; + BlobContainerClient destinationContainerClient = destinationDisposingContainer.Container; + await sourceContainerClient.CreateIfNotExistsAsync(); + await BlobTestSetupHelper.CreateBlobsInDirectoryAsync( + sourceContainerClient, + blobType, + pathPrefix, + _blobCount, + _blobSize, + cancellationToken); - string destinationContainerName = TestSetupHelper.Randomize("container"); - BlobContainerClient destinationContainerClient = _blobServiceClient.GetBlobContainerClient(destinationContainerName); - await destinationContainerClient.CreateIfNotExistsAsync(); + await destinationContainerClient.CreateIfNotExistsAsync(); - // Create Source Blob Container Storage Resource - StorageResource sourceResource = _blobsStorageResourceProvider.FromClient(sourceContainerClient, new() { BlobDirectoryPrefix = pathPrefix }); + // Create Source Blob Container Storage Resource + StorageResource sourceResource = _blobsStorageResourceProvider.FromClient(sourceContainerClient, new() { BlobDirectoryPrefix = pathPrefix }); - // Create Destination Blob Container Storage Resource - StorageResource destinationResource = _blobsStorageResourceProvider.FromClient( - destinationContainerClient, - new() + // Create Destination Blob Container Storage Resource + StorageResource destinationResource = _blobsStorageResourceProvider.FromClient( + destinationContainerClient, + new() + { + BlobDirectoryPrefix = pathPrefix, + BlobType = new(blobType) + }); + + // Start Transfer + await new TransferValidator() + { + TransferManager = new(_transferManagerOptions) + }.TransferAndVerifyAsync( + sourceResource, + destinationResource, + TransferValidator.GetBlobLister(sourceContainerClient, pathPrefix), + TransferValidator.GetBlobLister(destinationContainerClient, pathPrefix), + _blobCount, + _dataTransferOptions, + cancellationToken); + } + catch (TaskCanceledException) + { + // No action needed + } + catch (Exception ex) when + (ex is OutOfMemoryException + || ex is StackOverflowException + || ex is ThreadAbortException) { - BlobDirectoryPrefix = pathPrefix, - BlobType = new(blobType) - }); + throw; + } + catch (Exception ex) + { + _metrics.Client.GetMetric(Metrics.TransferRestarted).TrackValue(1); + _metrics.Client.TrackException(ex); + } + finally + { + // In case the container is the issue, delete the container. - // Start Transfer - await new TransferValidator() - { - TransferManager = new(_transferManagerOptions) - }.TransferAndVerifyAsync( - sourceResource, - destinationResource, - TransferValidator.GetBlobLister(sourceContainerClient, pathPrefix), - TransferValidator.GetBlobLister(destinationContainerClient, pathPrefix), - _blobCount, - _dataTransferOptions, - cancellationToken); + using var cancellationSource = new CancellationTokenSource(TimeSpan.FromSeconds(25)); + + try + { + if (sourceDisposingContainer != null) + { + _metrics.Client.TrackEvent("Stopping processing events"); + await sourceDisposingContainer.DisposeAsync(); + } + } + catch (Exception ex) + { + _metrics.Client.GetMetric(Metrics.TransferRestarted).TrackValue(1); + _metrics.Client.TrackException(ex); + } + } + } } } } diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyBlobSingleScenarioBase.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyBlobSingleScenarioBase.cs index a52013e28dc6..5e5f4afb9dcb 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyBlobSingleScenarioBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyBlobSingleScenarioBase.cs @@ -1,13 +1,16 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +extern alias BaseBlobs; + using System; using System.Threading; using System.Threading.Tasks; using Azure.Core; -using Azure.Storage.Blobs; -using Azure.Storage.Blobs.Models; -using Azure.Storage.Blobs.Specialized; +using BaseBlobs::Azure.Storage.Blobs; +using BaseBlobs::Azure.Storage.Blobs.Models; +using BaseBlobs::Azure.Storage.Blobs.Specialized; +using Azure.Storage.DataMovement.Blobs.Tests; using Azure.Storage.DataMovement.Tests; using Azure.Storage.Stress; @@ -34,74 +37,116 @@ public CopyBlobSingleScenarioBase( public async Task RunTestInternalAsync(BlobType blobType, CancellationToken cancellationToken) { - string sourceContainerName = TestSetupHelper.Randomize("container"); - BlobContainerClient sourceContainerClient = _sourceServiceClient.GetBlobContainerClient(sourceContainerName); - await sourceContainerClient.CreateIfNotExistsAsync(); + while (!cancellationToken.IsCancellationRequested) + { + string sourceContainerName = TestSetupHelper.Randomize("container"); + DisposingBlobContainer sourceDisposingContainer = new(_sourceServiceClient.GetBlobContainerClient(sourceContainerName)); + string destinationContainerName = TestSetupHelper.Randomize("container"); + DisposingBlobContainer destinationDisposingContainer = new(_blobServiceClient.GetBlobContainerClient(destinationContainerName)); + try + { + BlobContainerClient sourceContainerClient = sourceDisposingContainer.Container; + BlobContainerClient destinationContainerClient = destinationDisposingContainer.Container; - string destinationContainerName = TestSetupHelper.Randomize("container"); - BlobContainerClient destinationContainerClient = _blobServiceClient.GetBlobContainerClient(destinationContainerName); - await destinationContainerClient.CreateIfNotExistsAsync(); - string blobName = TestSetupHelper.Randomize("blob"); + string blobName = TestSetupHelper.Randomize("blob"); - // Create Source and Destination Storage Resource - BlobBaseClient sourceBaseBlob; - StorageResource sourceResource; - BlobBaseClient destinationBaseBlob; - StorageResource destinationResource; - if (blobType == BlobType.Append) - { - AppendBlobClient sourceBlob = sourceContainerClient.GetAppendBlobClient(blobName); - await BlobTestSetupHelper.CreateAppendBlobAsync( - sourceContainerClient.GetAppendBlobClient(blobName), - _blobSize, - cancellationToken); - sourceBaseBlob = sourceBlob; - sourceResource = _blobsStorageResourceProvider.FromClient(sourceBlob); + // Create Source and Destination Storage Resource + BlobBaseClient sourceBaseBlob; + StorageResource sourceResource; + BlobBaseClient destinationBaseBlob; + StorageResource destinationResource; + if (blobType == BlobType.Append) + { + AppendBlobClient sourceBlob = sourceContainerClient.GetAppendBlobClient(blobName); + await BlobTestSetupHelper.CreateAppendBlobAsync( + sourceContainerClient.GetAppendBlobClient(blobName), + _blobSize, + cancellationToken); + sourceBaseBlob = sourceBlob; + sourceResource = _blobsStorageResourceProvider.FromClient(sourceBlob); - AppendBlobClient destinationBlob = destinationContainerClient.GetAppendBlobClient(blobName); - destinationBaseBlob = destinationBlob; - destinationResource = _blobsStorageResourceProvider.FromClient(destinationBlob); - } - else if (blobType == BlobType.Page) - { - PageBlobClient sourceBlob = sourceContainerClient.GetPageBlobClient(blobName); - await BlobTestSetupHelper.CreatePageBlobAsync( - sourceContainerClient.GetPageBlobClient(blobName), - _blobSize, - cancellationToken); - sourceBaseBlob = sourceBlob; - sourceResource = _blobsStorageResourceProvider.FromClient(sourceBlob); + AppendBlobClient destinationBlob = destinationContainerClient.GetAppendBlobClient(blobName); + destinationBaseBlob = destinationBlob; + destinationResource = _blobsStorageResourceProvider.FromClient(destinationBlob); + } + else if (blobType == BlobType.Page) + { + PageBlobClient sourceBlob = sourceContainerClient.GetPageBlobClient(blobName); + await BlobTestSetupHelper.CreatePageBlobAsync( + sourceContainerClient.GetPageBlobClient(blobName), + _blobSize, + cancellationToken); + sourceBaseBlob = sourceBlob; + sourceResource = _blobsStorageResourceProvider.FromClient(sourceBlob); - PageBlobClient destinationBlob = destinationContainerClient.GetPageBlobClient(blobName); - destinationBaseBlob = destinationBlob; - destinationResource = _blobsStorageResourceProvider.FromClient(destinationBlob); - } - else - { - BlockBlobClient sourceBlob = sourceContainerClient.GetBlockBlobClient(blobName); - await BlobTestSetupHelper.CreateBlockBlobAsync( - sourceContainerClient.GetBlockBlobClient(blobName), - _blobSize, - cancellationToken); - sourceBaseBlob = sourceBlob; - sourceResource = _blobsStorageResourceProvider.FromClient(sourceBlob); + PageBlobClient destinationBlob = destinationContainerClient.GetPageBlobClient(blobName); + destinationBaseBlob = destinationBlob; + destinationResource = _blobsStorageResourceProvider.FromClient(destinationBlob); + } + else + { + BlockBlobClient sourceBlob = sourceContainerClient.GetBlockBlobClient(blobName); + await BlobTestSetupHelper.CreateBlockBlobAsync( + sourceContainerClient.GetBlockBlobClient(blobName), + _blobSize, + cancellationToken); + sourceBaseBlob = sourceBlob; + sourceResource = _blobsStorageResourceProvider.FromClient(sourceBlob); - BlockBlobClient destinationBlob = destinationContainerClient.GetBlockBlobClient(blobName); - destinationBaseBlob = destinationBlob; - destinationResource = _blobsStorageResourceProvider.FromClient(destinationBlob); - } + BlockBlobClient destinationBlob = destinationContainerClient.GetBlockBlobClient(blobName); + destinationBaseBlob = destinationBlob; + destinationResource = _blobsStorageResourceProvider.FromClient(destinationBlob); + } - // Start Transfer - await new TransferValidator() - { - TransferManager = new(_transferManagerOptions) - }.TransferAndVerifyAsync( - sourceResource, - destinationResource, - async cToken => await sourceBaseBlob.OpenReadAsync(default, cToken), - async cToken => await destinationBaseBlob.OpenReadAsync(default, cToken), - options: _dataTransferOptions, - cancellationToken: cancellationToken); + // Start Transfer + await new TransferValidator() + { + TransferManager = new(_transferManagerOptions) + }.TransferAndVerifyAsync( + sourceResource, + destinationResource, + async cToken => await sourceBaseBlob.OpenReadAsync(default, cToken), + async cToken => await destinationBaseBlob.OpenReadAsync(default, cToken), + options: _dataTransferOptions, + cancellationToken: cancellationToken); + } + catch (TaskCanceledException) + { + // No action needed + } + catch (Exception ex) when + (ex is OutOfMemoryException + || ex is StackOverflowException + || ex is ThreadAbortException) + { + throw; + } + catch (Exception ex) + { + _metrics.Client.GetMetric(Metrics.TransferRestarted).TrackValue(1); + _metrics.Client.TrackException(ex); + } + finally + { + // In case the container is the issue, delete the container. + + using var cancellationSource = new CancellationTokenSource(TimeSpan.FromSeconds(25)); + + try + { + if (sourceDisposingContainer != null) + { + _metrics.Client.TrackEvent("Stopping processing events"); + await sourceDisposingContainer.DisposeAsync(); + } + } + catch (Exception ex) + { + _metrics.Client.GetMetric(Metrics.TransferRestarted).TrackValue(1); + _metrics.Client.TrackException(ex); + } + } + } } } } diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyBlockBlobDirectoryScenario.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyBlockBlobDirectoryScenario.cs index 783c45004f11..227a4a92ae87 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyBlockBlobDirectoryScenario.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyBlockBlobDirectoryScenario.cs @@ -1,11 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +extern alias BaseBlobs; using System; using System.Threading; using System.Threading.Tasks; using Azure.Core; -using Azure.Storage.Blobs.Models; +using BaseBlobs::Azure.Storage.Blobs.Models; using Azure.Storage.Stress; namespace Azure.Storage.DataMovement.Blobs.Stress diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyBlockBlobSingleScenario.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyBlockBlobSingleScenario.cs index 906f713106bc..b8df500cc26f 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyBlockBlobSingleScenario.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyBlockBlobSingleScenario.cs @@ -1,11 +1,13 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +extern alias BaseBlobs; + using System; using System.Threading; using System.Threading.Tasks; using Azure.Core; -using Azure.Storage.Blobs.Models; +using BaseBlobs::Azure.Storage.Blobs.Models; using Azure.Storage.Stress; namespace Azure.Storage.DataMovement.Blobs.Stress diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyPageBlobDirectoryScenario.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyPageBlobDirectoryScenario.cs index d1776fcd22a4..99ea6940279e 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyPageBlobDirectoryScenario.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyPageBlobDirectoryScenario.cs @@ -1,11 +1,13 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +extern alias BaseBlobs; + using System; using System.Threading; using System.Threading.Tasks; using Azure.Core; -using Azure.Storage.Blobs.Models; +using BaseBlobs::Azure.Storage.Blobs.Models; using Azure.Storage.Stress; namespace Azure.Storage.DataMovement.Blobs.Stress diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyPageBlobSingleScenario.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyPageBlobSingleScenario.cs index 3ff37d142230..41bb2935480e 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyPageBlobSingleScenario.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyPageBlobSingleScenario.cs @@ -1,11 +1,13 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +extern alias BaseBlobs; + using System; using System.Threading; using System.Threading.Tasks; using Azure.Core; -using Azure.Storage.Blobs.Models; +using BaseBlobs::Azure.Storage.Blobs.Models; using Azure.Storage.Stress; namespace Azure.Storage.DataMovement.Blobs.Stress diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadAppendBlobDirectoryScenario.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadAppendBlobDirectoryScenario.cs index 3f5facb834bd..d29e0e02cce7 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadAppendBlobDirectoryScenario.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadAppendBlobDirectoryScenario.cs @@ -1,11 +1,13 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +extern alias BaseBlobs; + using System; using System.Threading; using System.Threading.Tasks; using Azure.Core; -using Azure.Storage.Blobs.Models; +using BaseBlobs::Azure.Storage.Blobs.Models; using Azure.Storage.Stress; namespace Azure.Storage.DataMovement.Blobs.Stress diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadAppendBlobSingleScenario.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadAppendBlobSingleScenario.cs index 079364cb8504..46e617e599ea 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadAppendBlobSingleScenario.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadAppendBlobSingleScenario.cs @@ -1,13 +1,15 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +extern alias BaseBlobs; + using System; using System.IO; using System.Threading; using System.Threading.Tasks; using Azure.Core; using Azure.Storage.Stress; -using Azure.Storage.Blobs.Models; +using BaseBlobs::Azure.Storage.Blobs.Models; namespace Azure.Storage.DataMovement.Blobs.Stress { diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadBlobDirectoryScenarioBase.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadBlobDirectoryScenarioBase.cs index bfb644feb2f2..647bd0c504a8 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadBlobDirectoryScenarioBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadBlobDirectoryScenarioBase.cs @@ -1,12 +1,15 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +extern alias BaseBlobs; + using System; using System.Threading; using System.Threading.Tasks; using Azure.Core; -using Azure.Storage.Blobs; -using Azure.Storage.Blobs.Models; +using BaseBlobs::Azure.Storage.Blobs; +using BaseBlobs::Azure.Storage.Blobs.Models; +using Azure.Storage.DataMovement.Blobs.Tests; using Azure.Storage.DataMovement.Tests; using Azure.Storage.Stress; @@ -31,42 +34,85 @@ public DownloadBlobDirectoryScenarioBase( public async Task RunTestInternalAsync(BlobType blobType, CancellationToken cancellationToken) { - // Create Source Blob Container - string sourceContainerName = TestSetupHelper.Randomize("container"); - BlobContainerClient sourceContainerClient = _blobServiceClient.GetBlobContainerClient(sourceContainerName); - await sourceContainerClient.CreateIfNotExistsAsync(cancellationToken: cancellationToken); + while (!cancellationToken.IsCancellationRequested) + { + string sourceContainerName = TestSetupHelper.Randomize("container"); + DisposingBlobContainer dipsosingContainer = new(_blobServiceClient.GetBlobContainerClient(sourceContainerName)); + try + { + // Create Source Blob Container + BlobContainerClient sourceContainerClient = dipsosingContainer.Container; + await sourceContainerClient.CreateIfNotExistsAsync(cancellationToken: cancellationToken); - // Create Destination Test Local Directory - string pathPrefix = TestSetupHelper.Randomize("dir"); - DisposingLocalDirectory disposingLocalDirectory = DisposingLocalDirectory.GetTestDirectory(pathPrefix); + // Create Destination Test Local Directory + string pathPrefix = TestSetupHelper.Randomize("dir"); + DisposingLocalDirectory disposingLocalDirectory = DisposingLocalDirectory.GetTestDirectory(pathPrefix); - // Create Blobs in source container - await BlobTestSetupHelper.CreateBlobsInDirectoryAsync( - sourceContainerClient, - blobType, - pathPrefix, - _blobCount, - _blobSize, - cancellationToken); + // Create Blobs in source container + await BlobTestSetupHelper.CreateBlobsInDirectoryAsync( + sourceContainerClient, + blobType, + pathPrefix, + _blobCount, + _blobSize, + cancellationToken); - // Create Destination Local Storage Resource - StorageResource sourceResource = await TestSetupHelper.GetTemporaryFileStorageResourceAsync(disposingLocalDirectory.DirectoryPath); + // Create Destination Local Storage Resource + StorageResource sourceResource = await TestSetupHelper.GetTemporaryFileStorageResourceAsync(disposingLocalDirectory.DirectoryPath); - // Create Destination Storage Resource - StorageResource destinationResource = _blobsStorageResourceProvider.FromClient(sourceContainerClient, new() { BlobDirectoryPrefix = pathPrefix }); + // Create Destination Storage Resource + StorageResource destinationResource = _blobsStorageResourceProvider.FromClient(sourceContainerClient, new() { BlobDirectoryPrefix = pathPrefix }); - // Start Transfer - await new TransferValidator() - { - TransferManager = new(_transferManagerOptions) - }.TransferAndVerifyAsync( - sourceResource, - destinationResource, - TransferValidator.GetBlobLister(sourceContainerClient, pathPrefix), - TransferValidator.GetLocalFileLister(disposingLocalDirectory.DirectoryPath), - _blobCount, - _dataTransferOptions, - cancellationToken); + // Start Transfer + await new TransferValidator() + { + TransferManager = new(_transferManagerOptions) + }.TransferAndVerifyAsync( + sourceResource, + destinationResource, + TransferValidator.GetBlobLister(sourceContainerClient, pathPrefix), + TransferValidator.GetLocalFileLister(disposingLocalDirectory.DirectoryPath), + _blobCount, + _dataTransferOptions, + cancellationToken); + } + catch (TaskCanceledException) + { + // No action needed + } + catch (Exception ex) when + (ex is OutOfMemoryException + || ex is StackOverflowException + || ex is ThreadAbortException) + { + throw; + } + catch (Exception ex) + { + _metrics.Client.GetMetric(Metrics.TransferRestarted).TrackValue(1); + _metrics.Client.TrackException(ex); + } + finally + { + // In case the container is the issue, delete the container. + + using var cancellationSource = new CancellationTokenSource(TimeSpan.FromSeconds(25)); + + try + { + if (dipsosingContainer != null) + { + _metrics.Client.TrackEvent("Stopping processing events"); + await dipsosingContainer.DisposeAsync(); + } + } + catch (Exception ex) + { + _metrics.Client.GetMetric(Metrics.TransferRestarted).TrackValue(1); + _metrics.Client.TrackException(ex); + } + } + } } } } diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadBlobSingleScenarioBase.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadBlobSingleScenarioBase.cs index 58a6f0dbf30a..bcc801b77957 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadBlobSingleScenarioBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadBlobSingleScenarioBase.cs @@ -1,16 +1,19 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +extern alias BaseBlobs; + using System; using System.IO; using System.Threading; using System.Threading.Tasks; using Azure.Core; using Azure.Storage.Stress; -using Azure.Storage.Blobs; -using Azure.Storage.Blobs.Specialized; +using BaseBlobs::Azure.Storage.Blobs; +using BaseBlobs::Azure.Storage.Blobs.Specialized; using Azure.Storage.DataMovement.Tests; -using Azure.Storage.Blobs.Models; +using BaseBlobs::Azure.Storage.Blobs.Models; +using Azure.Storage.DataMovement.Blobs.Tests; namespace Azure.Storage.DataMovement.Blobs.Stress { @@ -30,62 +33,105 @@ public DownloadBlobSingleScenarioBase( public async Task RunTestInternalAsync(BlobType blobType, CancellationToken cancellationToken) { - string sourceContainerName = TestSetupHelper.Randomize("container"); - BlobContainerClient sourceContainerClient = _blobServiceClient.GetBlobContainerClient(sourceContainerName); - await sourceContainerClient.CreateIfNotExistsAsync(cancellationToken: cancellationToken); + while (!cancellationToken.IsCancellationRequested) + { + string sourceContainerName = TestSetupHelper.Randomize("container"); + DisposingBlobContainer dipsosingContainer = new(_blobServiceClient.GetBlobContainerClient(sourceContainerName)); + try + { + BlobContainerClient sourceContainerClient = dipsosingContainer.Container; + await sourceContainerClient.CreateIfNotExistsAsync(cancellationToken: cancellationToken); - DisposingLocalDirectory disposingLocalDirectory = DisposingLocalDirectory.GetTestDirectory(); + DisposingLocalDirectory disposingLocalDirectory = DisposingLocalDirectory.GetTestDirectory(); - string blobName = TestSetupHelper.Randomize("blob"); + string blobName = TestSetupHelper.Randomize("blob"); - // Create Source Storage Resource - BlobBaseClient sourceBaseBlob; - StorageResource sourceResource; - if (blobType == BlobType.Append) - { - AppendBlobClient sourceBlob = sourceContainerClient.GetAppendBlobClient(blobName); - await BlobTestSetupHelper.CreateAppendBlobAsync( - sourceContainerClient.GetAppendBlobClient(blobName), - _blobSize, - cancellationToken); - sourceBaseBlob = sourceBlob; - sourceResource = _blobsStorageResourceProvider.FromClient(sourceBlob); - } - else if (blobType == BlobType.Page) - { - PageBlobClient sourceBlob = sourceContainerClient.GetPageBlobClient(blobName); - await BlobTestSetupHelper.CreatePageBlobAsync( - sourceContainerClient.GetPageBlobClient(blobName), - _blobSize, - cancellationToken); - sourceBaseBlob = sourceBlob; - sourceResource = _blobsStorageResourceProvider.FromClient(sourceBlob); - } - else - { - BlockBlobClient sourceBlob = sourceContainerClient.GetBlockBlobClient(blobName); - await BlobTestSetupHelper.CreateBlockBlobAsync( - sourceContainerClient.GetBlockBlobClient(blobName), - _blobSize, - cancellationToken); - sourceBaseBlob = sourceBlob; - sourceResource = _blobsStorageResourceProvider.FromClient(sourceBlob); - } + // Create Source Storage Resource + BlobBaseClient sourceBaseBlob; + StorageResource sourceResource; + if (blobType == BlobType.Append) + { + AppendBlobClient sourceBlob = sourceContainerClient.GetAppendBlobClient(blobName); + await BlobTestSetupHelper.CreateAppendBlobAsync( + sourceContainerClient.GetAppendBlobClient(blobName), + _blobSize, + cancellationToken); + sourceBaseBlob = sourceBlob; + sourceResource = _blobsStorageResourceProvider.FromClient(sourceBlob); + } + else if (blobType == BlobType.Page) + { + PageBlobClient sourceBlob = sourceContainerClient.GetPageBlobClient(blobName); + await BlobTestSetupHelper.CreatePageBlobAsync( + sourceContainerClient.GetPageBlobClient(blobName), + _blobSize, + cancellationToken); + sourceBaseBlob = sourceBlob; + sourceResource = _blobsStorageResourceProvider.FromClient(sourceBlob); + } + else + { + BlockBlobClient sourceBlob = sourceContainerClient.GetBlockBlobClient(blobName); + await BlobTestSetupHelper.CreateBlockBlobAsync( + sourceContainerClient.GetBlockBlobClient(blobName), + _blobSize, + cancellationToken); + sourceBaseBlob = sourceBlob; + sourceResource = _blobsStorageResourceProvider.FromClient(sourceBlob); + } - // Create Local Destination Storage Resource - StorageResource destinationResource = _localFilesStorageResourceProvider.FromFile(Path.Combine(disposingLocalDirectory.DirectoryPath, blobName)); + // Create Local Destination Storage Resource + StorageResource destinationResource = _localFilesStorageResourceProvider.FromFile(Path.Combine(disposingLocalDirectory.DirectoryPath, blobName)); - // Start Transfer - await new TransferValidator() - { - TransferManager = new(_transferManagerOptions) - }.TransferAndVerifyAsync( - sourceResource as StorageResourceItem, - destinationResource as StorageResourceItem, - async cToken => await sourceBaseBlob.OpenReadAsync(default, cToken), - cToken => Task.FromResult(File.OpenRead(sourceResource.Uri.AbsolutePath) as Stream), - options: _dataTransferOptions, - cancellationToken: cancellationToken); + // Start Transfer + await new TransferValidator() + { + TransferManager = new(_transferManagerOptions) + }.TransferAndVerifyAsync( + sourceResource as StorageResourceItem, + destinationResource as StorageResourceItem, + async cToken => await sourceBaseBlob.OpenReadAsync(default, cToken), + cToken => Task.FromResult(File.OpenRead(sourceResource.Uri.AbsolutePath) as Stream), + options: _dataTransferOptions, + cancellationToken: cancellationToken); + } + catch (TaskCanceledException) + { + // No action needed + } + catch (Exception ex) when + (ex is OutOfMemoryException + || ex is StackOverflowException + || ex is ThreadAbortException) + { + throw; + } + catch (Exception ex) + { + _metrics.Client.GetMetric(Metrics.TransferRestarted).TrackValue(1); + _metrics.Client.TrackException(ex); + } + finally + { + // In case the container is the issue, delete the container. + + using var cancellationSource = new CancellationTokenSource(TimeSpan.FromSeconds(25)); + + try + { + if (dipsosingContainer != null) + { + _metrics.Client.TrackEvent("Stopping processing events"); + await dipsosingContainer.DisposeAsync(); + } + } + catch (Exception ex) + { + _metrics.Client.GetMetric(Metrics.TransferRestarted).TrackValue(1); + _metrics.Client.TrackException(ex); + } + } + } } } } diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadBlockBlobDirectoryScenario.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadBlockBlobDirectoryScenario.cs index 331ca15ec302..a241d9b7f84b 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadBlockBlobDirectoryScenario.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadBlockBlobDirectoryScenario.cs @@ -1,11 +1,13 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +extern alias BaseBlobs; + using System; using System.Threading; using System.Threading.Tasks; using Azure.Core; -using Azure.Storage.Blobs.Models; +using BaseBlobs::Azure.Storage.Blobs.Models; using Azure.Storage.Stress; namespace Azure.Storage.DataMovement.Blobs.Stress diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadBlockBlobSingleScenario.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadBlockBlobSingleScenario.cs index 6e02ca1a4c47..523cd4cac314 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadBlockBlobSingleScenario.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadBlockBlobSingleScenario.cs @@ -1,13 +1,15 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +extern alias BaseBlobs; + using System; using System.IO; using System.Threading; using System.Threading.Tasks; using Azure.Core; using Azure.Storage.Stress; -using Azure.Storage.Blobs.Models; +using BaseBlobs::Azure.Storage.Blobs.Models; namespace Azure.Storage.DataMovement.Blobs.Stress { diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadPageBlobDirectoryScenario.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadPageBlobDirectoryScenario.cs index 41ee4a9687b5..03eedc752246 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadPageBlobDirectoryScenario.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadPageBlobDirectoryScenario.cs @@ -1,11 +1,13 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +extern alias BaseBlobs; + using System; using System.Threading; using System.Threading.Tasks; using Azure.Core; -using Azure.Storage.Blobs.Models; +using BaseBlobs::Azure.Storage.Blobs.Models; using Azure.Storage.Stress; namespace Azure.Storage.DataMovement.Blobs.Stress diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadPageBlobSingleScenario.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadPageBlobSingleScenario.cs index 7088f1ce5d0a..9f287eb57fae 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadPageBlobSingleScenario.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/DownloadPageBlobSingleScenario.cs @@ -1,13 +1,15 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +extern alias BaseBlobs; + using System; using System.IO; using System.Threading; using System.Threading.Tasks; using Azure.Core; using Azure.Storage.Stress; -using Azure.Storage.Blobs.Models; +using BaseBlobs::Azure.Storage.Blobs.Models; namespace Azure.Storage.DataMovement.Blobs.Stress { diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadAppendBlobDirectoryScenario.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadAppendBlobDirectoryScenario.cs index 8b7f63f9dddb..c2c24f740280 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadAppendBlobDirectoryScenario.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadAppendBlobDirectoryScenario.cs @@ -1,12 +1,14 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +extern alias BaseBlobs; + using System; using System.Threading; using System.Threading.Tasks; using Azure.Core; using Azure.Storage.Stress; -using Azure.Storage.Blobs.Models; +using BaseBlobs::Azure.Storage.Blobs.Models; namespace Azure.Storage.DataMovement.Blobs.Stress { diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadAppendBlobSingleScenario.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadAppendBlobSingleScenario.cs index 9aaabb5a3885..e5884ac717ab 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadAppendBlobSingleScenario.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadAppendBlobSingleScenario.cs @@ -1,12 +1,14 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +extern alias BaseBlobs; + using System; using System.Threading; using System.Threading.Tasks; using Azure.Core; using Azure.Storage.Stress; -using Azure.Storage.Blobs.Models; +using BaseBlobs::Azure.Storage.Blobs.Models; namespace Azure.Storage.DataMovement.Blobs.Stress { diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadBlobDirectoryScenarioBase.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadBlobDirectoryScenarioBase.cs index 9db7441966d1..e4b8d7cb51f2 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadBlobDirectoryScenarioBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadBlobDirectoryScenarioBase.cs @@ -1,14 +1,17 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +extern alias BaseBlobs; + using System; using System.Threading; using System.Threading.Tasks; using Azure.Core; using Azure.Storage.Stress; -using Azure.Storage.Blobs; -using Azure.Storage.Blobs.Models; +using BaseBlobs::Azure.Storage.Blobs; +using BaseBlobs::Azure.Storage.Blobs.Models; using Azure.Storage.DataMovement.Tests; +using Azure.Storage.DataMovement.Blobs.Tests; namespace Azure.Storage.DataMovement.Blobs.Stress { @@ -31,45 +34,88 @@ public UploadBlobDirectoryScenarioBase( public async Task RunTestInternalAsync(BlobType blobType, CancellationToken cancellationToken) { - // Create test local directory - string pathPrefix = TestSetupHelper.Randomize("dir"); - DisposingLocalDirectory disposingLocalDirectory = DisposingLocalDirectory.GetTestDirectory(pathPrefix); + while (!cancellationToken.IsCancellationRequested) + { + string destinationContainerName = TestSetupHelper.Randomize("container"); + DisposingBlobContainer dipsosingContainer = new(_blobServiceClient.GetBlobContainerClient(destinationContainerName)); + try + { + // Create test local directory + string pathPrefix = TestSetupHelper.Randomize("dir"); + DisposingLocalDirectory disposingLocalDirectory = DisposingLocalDirectory.GetTestDirectory(pathPrefix); - // Create destination blob container - string destinationContainerName = TestSetupHelper.Randomize("container"); - BlobContainerClient destinationContainerClient = _blobServiceClient.GetBlobContainerClient(destinationContainerName); - await destinationContainerClient.CreateIfNotExistsAsync(); + // Create destination blob container + BlobContainerClient destinationContainerClient = dipsosingContainer.Container; + await destinationContainerClient.CreateIfNotExistsAsync(); - // Create Local Files - await TestSetupHelper.CreateLocalFilesToUploadAsync( - disposingLocalDirectory.DirectoryPath, - _blobCount, - _blobSize); + // Create Local Files + await TestSetupHelper.CreateLocalFilesToUploadAsync( + disposingLocalDirectory.DirectoryPath, + _blobCount, + _blobSize); - // Create Local Source Storage Resource - StorageResource sourceResource = await TestSetupHelper.GetTemporaryFileStorageResourceAsync(disposingLocalDirectory.DirectoryPath); + // Create Local Source Storage Resource + StorageResource sourceResource = await TestSetupHelper.GetTemporaryFileStorageResourceAsync(disposingLocalDirectory.DirectoryPath); - // Create Destination Storage Resource - StorageResource destinationResource = _blobsStorageResourceProvider.FromClient( - destinationContainerClient, - new() + // Create Destination Storage Resource + StorageResource destinationResource = _blobsStorageResourceProvider.FromClient( + destinationContainerClient, + new() + { + BlobDirectoryPrefix = pathPrefix, + BlobType = new(blobType) + }); + + // Upload + await new TransferValidator() + { + TransferManager = new(_transferManagerOptions) + }.TransferAndVerifyAsync( + sourceResource, + destinationResource, + TransferValidator.GetLocalFileLister(disposingLocalDirectory.DirectoryPath), + TransferValidator.GetBlobLister(destinationContainerClient, default), + _blobCount, + _dataTransferOptions, + cancellationToken); + } + catch (TaskCanceledException) + { + // No action needed + } + catch (Exception ex) when + (ex is OutOfMemoryException + || ex is StackOverflowException + || ex is ThreadAbortException) { - BlobDirectoryPrefix = pathPrefix, - BlobType = new(blobType) - }); + throw; + } + catch (Exception ex) + { + _metrics.Client.GetMetric(Metrics.TransferRestarted).TrackValue(1); + _metrics.Client.TrackException(ex); + } + finally + { + // In case the container is the issue, delete the container. - // Upload - await new TransferValidator() - { - TransferManager = new(_transferManagerOptions) - }.TransferAndVerifyAsync( - sourceResource, - destinationResource, - TransferValidator.GetLocalFileLister(disposingLocalDirectory.DirectoryPath), - TransferValidator.GetBlobLister(destinationContainerClient, default), - _blobCount, - _dataTransferOptions, - cancellationToken); + using var cancellationSource = new CancellationTokenSource(TimeSpan.FromSeconds(25)); + + try + { + if (dipsosingContainer != null) + { + _metrics.Client.TrackEvent("Stopping processing events"); + await dipsosingContainer.DisposeAsync(); + } + } + catch (Exception ex) + { + _metrics.Client.GetMetric(Metrics.TransferRestarted).TrackValue(1); + _metrics.Client.TrackException(ex); + } + } + } } } } diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadBlobSingleScenarioBase.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadBlobSingleScenarioBase.cs index f45c27bfd3f1..4e9c12e089c3 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadBlobSingleScenarioBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadBlobSingleScenarioBase.cs @@ -1,16 +1,19 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +extern alias BaseBlobs; + using System; using System.IO; using System.Threading; using System.Threading.Tasks; using Azure.Core; using Azure.Storage.Stress; -using Azure.Storage.Blobs; -using Azure.Storage.Blobs.Specialized; +using BaseBlobs::Azure.Storage.Blobs; +using BaseBlobs::Azure.Storage.Blobs.Specialized; using Azure.Storage.DataMovement.Tests; -using Azure.Storage.Blobs.Models; +using BaseBlobs::Azure.Storage.Blobs.Models; +using Azure.Storage.DataMovement.Blobs.Tests; namespace Azure.Storage.DataMovement.Blobs.Stress { @@ -29,52 +32,95 @@ protected UploadBlobSingleScenarioBase( internal async Task RunTestInternalAsync(BlobType blobType, CancellationToken cancellationToken = default) { - DisposingLocalDirectory disposingLocalDirectory = DisposingLocalDirectory.GetTestDirectory(); - string destinationContainerName = TestSetupHelper.Randomize("container"); - BlobContainerClient destinationContainerClient = _blobServiceClient.GetBlobContainerClient(destinationContainerName); - await destinationContainerClient.CreateIfNotExistsAsync(); - string blobName = TestSetupHelper.Randomize("blob"); + while (!cancellationToken.IsCancellationRequested) + { + string destinationContainerName = TestSetupHelper.Randomize("container"); + DisposingBlobContainer dipsosingContainer = new(_blobServiceClient.GetBlobContainerClient(destinationContainerName)); + try + { + DisposingLocalDirectory disposingLocalDirectory = DisposingLocalDirectory.GetTestDirectory(); + BlobContainerClient destinationContainerClient = dipsosingContainer.Container; + await destinationContainerClient.CreateIfNotExistsAsync(); + string blobName = TestSetupHelper.Randomize("blob"); - // Create Local Source Storage Resource - StorageResource sourceResource = await TestSetupHelper.GetTemporaryFileStorageResourceAsync( - disposingLocalDirectory.DirectoryPath, - fileName: blobName, - fileSize: _blobSize, - cancellationToken: cancellationToken); + // Create Local Source Storage Resource + StorageResource sourceResource = await TestSetupHelper.GetTemporaryFileStorageResourceAsync( + disposingLocalDirectory.DirectoryPath, + fileName: blobName, + fileSize: _blobSize, + cancellationToken: cancellationToken); - // Create Destination Storage Resource - BlobBaseClient destinationBaseBlob; - StorageResource destinationResource; - if (blobType == BlobType.Append) - { - AppendBlobClient destinationBlob = destinationContainerClient.GetAppendBlobClient(blobName); - destinationBaseBlob = destinationBlob; - destinationResource = _blobsStorageResourceProvider.FromClient(destinationBlob); - } - else if (blobType == BlobType.Page) - { - PageBlobClient destinationBlob = destinationContainerClient.GetPageBlobClient(blobName); - destinationBaseBlob = destinationBlob; - destinationResource = _blobsStorageResourceProvider.FromClient(destinationBlob); - } - else - { - BlockBlobClient destinationBlob = destinationContainerClient.GetBlockBlobClient(blobName); - destinationBaseBlob = destinationBlob; - destinationResource = _blobsStorageResourceProvider.FromClient(destinationBlob); - } + // Create Destination Storage Resource + BlobBaseClient destinationBaseBlob; + StorageResource destinationResource; + if (blobType == BlobType.Append) + { + AppendBlobClient destinationBlob = destinationContainerClient.GetAppendBlobClient(blobName); + destinationBaseBlob = destinationBlob; + destinationResource = _blobsStorageResourceProvider.FromClient(destinationBlob); + } + else if (blobType == BlobType.Page) + { + PageBlobClient destinationBlob = destinationContainerClient.GetPageBlobClient(blobName); + destinationBaseBlob = destinationBlob; + destinationResource = _blobsStorageResourceProvider.FromClient(destinationBlob); + } + else + { + BlockBlobClient destinationBlob = destinationContainerClient.GetBlockBlobClient(blobName); + destinationBaseBlob = destinationBlob; + destinationResource = _blobsStorageResourceProvider.FromClient(destinationBlob); + } - // Start Transfer - await new TransferValidator() - { - TransferManager = new(_transferManagerOptions) - }.TransferAndVerifyAsync( - sourceResource, - destinationResource, - cToken => Task.FromResult(File.OpenRead(sourceResource.Uri.AbsolutePath) as Stream), - async cToken => await destinationBaseBlob.OpenReadAsync(default, cToken), - options: _dataTransferOptions, - cancellationToken: cancellationToken); + // Start Transfer + await new TransferValidator() + { + TransferManager = new(_transferManagerOptions) + }.TransferAndVerifyAsync( + sourceResource, + destinationResource, + cToken => Task.FromResult(File.OpenRead(sourceResource.Uri.AbsolutePath) as Stream), + async cToken => await destinationBaseBlob.OpenReadAsync(default, cToken), + options: _dataTransferOptions, + cancellationToken: cancellationToken); + } + catch (TaskCanceledException) + { + // No action needed + } + catch (Exception ex) when + (ex is OutOfMemoryException + || ex is StackOverflowException + || ex is ThreadAbortException) + { + throw; + } + catch (Exception ex) + { + _metrics.Client.GetMetric(Metrics.TransferRestarted).TrackValue(1); + _metrics.Client.TrackException(ex); + } + finally + { + // In case the container is the issue, delete the container. + + using var cancellationSource = new CancellationTokenSource(TimeSpan.FromSeconds(25)); + + try + { + if (dipsosingContainer != null) + { + _metrics.Client.TrackEvent("Stopping processing events"); + await dipsosingContainer.DisposeAsync(); + } + } + catch (Exception ex) + { + _metrics.Client.GetMetric(Metrics.TransferRestarted).TrackValue(1); + _metrics.Client.TrackException(ex); + } + } + } } } } diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadBlockBlobDirectoryScenario.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadBlockBlobDirectoryScenario.cs index 6a884a66875c..bfd87e26b08e 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadBlockBlobDirectoryScenario.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadBlockBlobDirectoryScenario.cs @@ -1,12 +1,14 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +extern alias BaseBlobs; + using System; using System.Threading; using System.Threading.Tasks; using Azure.Core; using Azure.Storage.Stress; -using Azure.Storage.Blobs.Models; +using BaseBlobs::Azure.Storage.Blobs.Models; namespace Azure.Storage.DataMovement.Blobs.Stress { diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadBlockBlobSingleScenario.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadBlockBlobSingleScenario.cs index 7d59645355ac..dca94ee847f0 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadBlockBlobSingleScenario.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadBlockBlobSingleScenario.cs @@ -1,12 +1,14 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +extern alias BaseBlobs; + using System; using System.Threading; using System.Threading.Tasks; using Azure.Core; using Azure.Storage.Stress; -using Azure.Storage.Blobs.Models; +using BaseBlobs::Azure.Storage.Blobs.Models; namespace Azure.Storage.DataMovement.Blobs.Stress { diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadPageBlobDirectoryScenario.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadPageBlobDirectoryScenario.cs index 985610a3e219..4979a513974a 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadPageBlobDirectoryScenario.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadPageBlobDirectoryScenario.cs @@ -1,12 +1,14 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +extern alias BaseBlobs; + using System; using System.Threading; using System.Threading.Tasks; using Azure.Core; using Azure.Storage.Stress; -using Azure.Storage.Blobs.Models; +using BaseBlobs::Azure.Storage.Blobs.Models; namespace Azure.Storage.DataMovement.Blobs.Stress { diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadPageBlobSingleScenario.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadPageBlobSingleScenario.cs index b70bb2a290cb..365bfef33def 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadPageBlobSingleScenario.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/UploadPageBlobSingleScenario.cs @@ -1,12 +1,14 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +extern alias BaseBlobs; + using System; using System.Threading; using System.Threading.Tasks; using Azure.Core; using Azure.Storage.Stress; -using Azure.Storage.Blobs.Models; +using BaseBlobs::Azure.Storage.Blobs.Models; namespace Azure.Storage.DataMovement.Blobs.Stress { diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/src/DataMovementBlobStressConstants.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/src/DataMovementBlobStressConstants.cs index 9e4d13bef614..1b23d9401e9b 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/src/DataMovementBlobStressConstants.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/src/DataMovementBlobStressConstants.cs @@ -21,10 +21,17 @@ public static partial class EnvironmentVariables /// /// The name of the environment variable that holds the connection string for the - /// Event Hubs Namespace resource for the test runs. + /// Storage Blob Namespace resource for the test runs. /// /// - public const string StorageBlobEndpoint = "STRESS_STORAGE_BLOB_ENDPOINT"; + public const string StorageSourceBlobEndpoint = "STRESS_STORAGE_SRC_BLOB_ENDPOINT"; + + /// + /// The name of the environment variable that holds the connection string for the + /// Storage Blob Namespace resource for the test runs. + /// + /// + public const string StorageDestinationBlobEndpoint = "STRESS_STORAGE_DEST_BLOB_ENDPOINT"; // Job Index Information diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/src/Program.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/src/Program.cs index fd4c16d3af8a..9e2bb2481bb8 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/src/Program.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/src/Program.cs @@ -44,7 +44,8 @@ private static async Task RunOptions(Options opts) environment = EnvironmentReader.LoadFromFile(environmentFile); environment.TryGetValue(DataMovementBlobStressConstants.EnvironmentVariables.ApplicationInsightsKey, out var appInsightsKey); - environment.TryGetValue(DataMovementBlobStressConstants.EnvironmentVariables.StorageBlobEndpoint, out var blobEndpoint); + environment.TryGetValue(DataMovementBlobStressConstants.EnvironmentVariables.StorageSourceBlobEndpoint, out var blobSourceEndpoint); + environment.TryGetValue(DataMovementBlobStressConstants.EnvironmentVariables.StorageDestinationBlobEndpoint, out var blobDestinationEndpoint); // Check values @@ -83,7 +84,7 @@ private static async Task RunOptions(Options opts) { case TestScenarioName.UploadSingleBlockBlob: testScenario = new UploadBlockBlobSingleScenario( - new Uri(blobEndpoint), + new Uri(blobSourceEndpoint), opts.Size, transferManagerOptions, transferOptions, @@ -93,7 +94,7 @@ private static async Task RunOptions(Options opts) break; case TestScenarioName.UploadDirectoryBlockBlob: testScenario = new UploadBlockBlobDirectoryScenario( - destinationBlobUri: new Uri(blobEndpoint), + destinationBlobUri: new Uri(blobSourceEndpoint), blobSize: opts.Size, blobCount: opts.Count, transferManagerOptions: transferManagerOptions, @@ -104,7 +105,7 @@ private static async Task RunOptions(Options opts) break; case TestScenarioName.DownloadSingleBlockBlob: testScenario = new DownloadBlockBlobSingleScenario( - new Uri(blobEndpoint), + new Uri(blobSourceEndpoint), opts.Size, transferManagerOptions, transferOptions, @@ -114,7 +115,7 @@ private static async Task RunOptions(Options opts) break; case TestScenarioName.DownloadDirectoryBlockBlob: testScenario = new DownloadBlockBlobDirectoryScenario( - new Uri(blobEndpoint), + new Uri(blobSourceEndpoint), opts.Size, opts.Count, transferManagerOptions, @@ -125,8 +126,8 @@ private static async Task RunOptions(Options opts) break; case TestScenarioName.CopySingleBlockBlob: testScenario = new CopyBlockBlobSingleScenario( - new Uri(blobEndpoint), - new Uri(blobEndpoint), + new Uri(blobSourceEndpoint), + new Uri(blobDestinationEndpoint), opts.Size, transferManagerOptions, transferOptions, @@ -137,8 +138,8 @@ private static async Task RunOptions(Options opts) break; case TestScenarioName.CopyDirectoryBlockBlob: testScenario = new CopyBlockBlobDirectoryScenario( - new Uri(blobEndpoint), - new Uri(blobEndpoint), + new Uri(blobSourceEndpoint), + new Uri(blobDestinationEndpoint), opts.Size, opts.Count, transferManagerOptions, @@ -150,7 +151,7 @@ private static async Task RunOptions(Options opts) break; case TestScenarioName.UploadSingleAppendBlob: testScenario = new UploadAppendBlobSingleScenario( - new Uri(blobEndpoint), + new Uri(blobSourceEndpoint), opts.Size, transferManagerOptions, transferOptions, @@ -160,7 +161,7 @@ private static async Task RunOptions(Options opts) break; case TestScenarioName.UploadDirectoryAppendBlob: testScenario = new UploadAppendBlobDirectoryScenario( - new Uri(blobEndpoint), + new Uri(blobSourceEndpoint), opts.Size, opts.Count, transferManagerOptions, @@ -171,7 +172,7 @@ private static async Task RunOptions(Options opts) break; case TestScenarioName.DownloadSingleAppendBlob: testScenario = new DownloadAppendBlobSingleScenario( - new Uri(blobEndpoint), + new Uri(blobSourceEndpoint), opts.Size, transferManagerOptions, transferOptions, @@ -181,7 +182,7 @@ private static async Task RunOptions(Options opts) break; case TestScenarioName.DownloadDirectoryAppendBlob: testScenario = new DownloadAppendBlobDirectoryScenario( - new Uri(blobEndpoint), + new Uri(blobSourceEndpoint), opts.Size, opts.Count, transferManagerOptions, @@ -192,8 +193,8 @@ private static async Task RunOptions(Options opts) break; case TestScenarioName.CopySingleAppendBlob: testScenario = new CopyAppendBlobSingleScenario( - new Uri(blobEndpoint), - new Uri(blobEndpoint), + new Uri(blobSourceEndpoint), + new Uri(blobDestinationEndpoint), opts.Size, transferManagerOptions, transferOptions, @@ -204,8 +205,8 @@ private static async Task RunOptions(Options opts) break; case TestScenarioName.CopyDirectoryAppendBlob: testScenario = new CopyAppendBlobDirectoryScenario( - new Uri(blobEndpoint), - new Uri(blobEndpoint), + new Uri(blobSourceEndpoint), + new Uri(blobDestinationEndpoint), opts.Size, opts.Count, transferManagerOptions, @@ -217,7 +218,7 @@ private static async Task RunOptions(Options opts) break; case TestScenarioName.UploadSinglePageBlob: testScenario = new UploadPageBlobSingleScenario( - new Uri(blobEndpoint), + new Uri(blobSourceEndpoint), opts.Size, transferManagerOptions, transferOptions, @@ -227,7 +228,7 @@ private static async Task RunOptions(Options opts) break; case TestScenarioName.UploadDirectoryPageBlob: testScenario = new UploadPageBlobDirectoryScenario( - new Uri(blobEndpoint), + new Uri(blobSourceEndpoint), opts.Size, opts.Count, transferManagerOptions, @@ -238,7 +239,7 @@ private static async Task RunOptions(Options opts) break; case TestScenarioName.DownloadSinglePageBlob: testScenario = new DownloadPageBlobSingleScenario( - new Uri(blobEndpoint), + new Uri(blobSourceEndpoint), opts.Size, transferManagerOptions, transferOptions, @@ -248,7 +249,7 @@ private static async Task RunOptions(Options opts) break; case TestScenarioName.DownloadDirectoryPageBlob: testScenario = new DownloadPageBlobDirectoryScenario( - new Uri(blobEndpoint), + new Uri(blobSourceEndpoint), opts.Size, opts.Count, transferManagerOptions, @@ -259,8 +260,8 @@ private static async Task RunOptions(Options opts) break; case TestScenarioName.CopySinglePageBlob: testScenario = new CopyPageBlobSingleScenario( - new Uri(blobEndpoint), - new Uri(blobEndpoint), + new Uri(blobSourceEndpoint), + new Uri(blobDestinationEndpoint), opts.Size, transferManagerOptions, transferOptions, @@ -271,8 +272,8 @@ private static async Task RunOptions(Options opts) break; case TestScenarioName.CopyDirectoryPageBlob: testScenario = new CopyPageBlobDirectoryScenario( - new Uri(blobEndpoint), - new Uri(blobEndpoint), + new Uri(blobSourceEndpoint), + new Uri(blobDestinationEndpoint), opts.Size, opts.Count, transferManagerOptions, diff --git a/sdk/storage/Azure.Storage.DataMovement/stress/DataMovementScenarioBase.cs b/sdk/storage/Azure.Storage.DataMovement/stress/DataMovementScenarioBase.cs new file mode 100644 index 000000000000..0ebe6ae2429d --- /dev/null +++ b/sdk/storage/Azure.Storage.DataMovement/stress/DataMovementScenarioBase.cs @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Threading.Tasks; +using Azure.Storage.Stress; + +namespace Azure.Storage.DataMovement.Stress; + +/// +/// The test scenario responsible for running all of the roles needed for the send receive test scenario. +/// +/// +public abstract class DataMovementScenarioBase : TestScenarioBase, IDisposable +{ + protected internal readonly TransferManagerOptions _transferManagerOptions; + protected internal readonly DataTransferOptions _dataTransferOptions; + + public DataMovementScenarioBase( + TransferManagerOptions transferManagerOptions, + DataTransferOptions dataTransferOptions, + Metrics metrics, + string testRunId) + : base(metrics, testRunId) + { + _transferManagerOptions = transferManagerOptions; + _dataTransferOptions = dataTransferOptions; + + // Add metric call backs when transfer has updated + _dataTransferOptions.TransferStatusChanged += AddStatusMetricArg; + _dataTransferOptions.ItemTransferFailed += AddFailedMetricArg; + _dataTransferOptions.ItemTransferCompleted += AddCompletedItemMetricArg; + _dataTransferOptions.ItemTransferSkipped += AddSkippedItemMetricArg; + } + + public void Dispose() + { + _dataTransferOptions.TransferStatusChanged -= AddStatusMetricArg; + _dataTransferOptions.ItemTransferFailed -= AddFailedMetricArg; + _dataTransferOptions.ItemTransferCompleted -= AddCompletedItemMetricArg; + _dataTransferOptions.ItemTransferSkipped -= AddSkippedItemMetricArg; + } + + private Task AddFailedMetricArg(TransferItemFailedEventArgs args) + { + _metrics.Client.GetMetric(Metrics.TransferFailedItem).TrackValue(1); + return Task.CompletedTask; + } + + private Task AddStatusMetricArg(TransferStatusEventArgs args) + { + _metrics.Client.GetMetric(Metrics.TransferStatusChanged).TrackValue(args.TransferStatus); + return Task.CompletedTask; + } + + private Task AddCompletedItemMetricArg(TransferItemCompletedEventArgs args) + { + _metrics.Client.GetMetric(Metrics.ItemTransferCompleted).TrackValue(1); + return Task.CompletedTask; + } + + private Task AddSkippedItemMetricArg(TransferItemSkippedEventArgs args) + { + _metrics.Client.GetMetric(Metrics.ItemTransferSkipped).TrackValue(1); + return Task.CompletedTask; + } +}