Skip to content

Commit bc3d564

Browse files
amnguyeamnguye
authored
[Storage] [DataMovement] [Blobs] Update to Stress Metrics (#46953)
* WIP * WIP * WIP - added metrics to DataTransferOptionsCallbacks * WIP * Update to scenario dependencies
1 parent b9342e8 commit bc3d564

34 files changed

+759
-350
lines changed

‎sdk/storage/Azure.Storage.Common/src/stress/Shared/Metrics.cs‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,14 +45,29 @@ public class Metrics
4545
/// This is the metric name used to collect how many objects (e.g. local file, blob, share file) were transferred.
4646
/// </summary>
4747
///
48-
public const string ObjectsTransferred = "ObjectsTransferred";
48+
public const string ItemTransferCompleted = "ItemTransferCompleted";
49+
50+
/// <summary>
51+
/// This is the metric name used to collect each transfer status that occurred.
52+
/// </summary>
53+
public const string TransferStatusChanged = "TransferStatusChanged";
54+
55+
/// <summary>
56+
/// This is the metric name used to keep track of the failed transfer items.
57+
/// </summary>
58+
public const string TransferFailedItem = "TransferFailedItem";
59+
60+
/// <summary>
61+
/// This is the metric name used to keep track of the skipped transfer items
62+
/// </summary>
63+
public const string ItemTransferSkipped = "ItemTransferSkipped";
4964

5065
/// <summary>
5166
/// This is the metric name used to collect metrics on how many times
5267
/// the individual transfer part was restarted or retried.
5368
/// </summary>
5469
///
55-
public const string TransferRetriedRestarted = "TransferRestarted";
70+
public const string TransferRestarted = "TransferRestarted";
5671

5772
/// <summary>
5873
/// This is the metric name used to collect metrics on the total number of bytes that have been

‎sdk/storage/Azure.Storage.Common/src/stress/Shared/TestScenarioBase.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ public abstract class TestScenarioBase
2121
/// <summary>
2222
/// Initializes a new Test instance.
2323
/// </summary>
24-
///\
24+
///
2525
/// <param name="metrics">The <see cref="Metrics" /> to use to send metrics to Application Insights.</param>
2626
/// <param name="testRunId">Test Run Id to differ between test runs.</param>
2727
///

‎sdk/storage/Azure.Storage.DataMovement.Blobs/stress/generatedValues.yaml‎

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,92 +1,92 @@
11
scenarios:
22
- image: Dockerfile
3-
imageBuildDir: ../../../..
43
Scenario: uploadsingleblockblob
54
testScenario: uploadsingleblockblob
5+
imageBuildDir: ../../../..
66
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
77
- image: Dockerfile
8+
Scenario: uploaddirectoryblockblob
9+
testScenario: uploaddirectoryblockblob
810
imageBuildDir: ../../../..
9-
Scenario: uploaddirectoryblockBlob
10-
testScenario: uploaddirectoryblockBlob
1111
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
1212
- image: Dockerfile
13-
imageBuildDir: ../../../..
1413
Scenario: downloadsingleblockblob
1514
testScenario: downloadsingleblockblob
15+
imageBuildDir: ../../../..
1616
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
1717
- image: Dockerfile
18-
imageBuildDir: ../../../..
1918
Scenario: downloaddirectoryblockblob
2019
testScenario: downloaddirectoryblockblob
20+
imageBuildDir: ../../../..
2121
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
2222
- image: Dockerfile
23-
imageBuildDir: ../../../..
2423
Scenario: copysingleblockblob
2524
testScenario: copysingleblockblob
25+
imageBuildDir: ../../../..
2626
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
2727
- image: Dockerfile
28-
imageBuildDir: ../../../..
2928
Scenario: copydirectoryblockblob
3029
testScenario: copydirectoryblockblob
30+
imageBuildDir: ../../../..
3131
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
3232
- image: Dockerfile
33-
imageBuildDir: ../../../..
3433
Scenario: uploadsingleappendblob
3534
testScenario: uploadsingleappendblob
35+
imageBuildDir: ../../../..
3636
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
3737
- image: Dockerfile
38-
imageBuildDir: ../../../..
3938
Scenario: uploaddirectoryappendblob
4039
testScenario: uploaddirectoryappendblob
40+
imageBuildDir: ../../../..
4141
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
4242
- image: Dockerfile
43-
imageBuildDir: ../../../..
4443
Scenario: downloadsingleappendblob
4544
testScenario: downloadsingleappendblob
45+
imageBuildDir: ../../../..
4646
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
4747
- image: Dockerfile
48-
imageBuildDir: ../../../..
4948
Scenario: downloaddirectoryappendblob
5049
testScenario: downloaddirectoryappendblob
50+
imageBuildDir: ../../../..
5151
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
5252
- image: Dockerfile
53-
imageBuildDir: ../../../..
5453
Scenario: copysingleappendblob
5554
testScenario: copysingleappendblob
55+
imageBuildDir: ../../../..
5656
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
5757
- image: Dockerfile
58-
imageBuildDir: ../../../..
5958
Scenario: copydirectoryappendblob
6059
testScenario: copydirectoryappendblob
60+
imageBuildDir: ../../../..
6161
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
6262
- image: Dockerfile
63-
imageBuildDir: ../../../..
6463
Scenario: uploadsinglepageblob
6564
testScenario: uploadsinglepageblob
65+
imageBuildDir: ../../../..
6666
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
6767
- image: Dockerfile
68-
imageBuildDir: ../../../..
6968
Scenario: uploaddirectorypageblob
7069
testScenario: uploaddirectorypageblob
70+
imageBuildDir: ../../../..
7171
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
7272
- image: Dockerfile
73-
imageBuildDir: ../../../..
7473
Scenario: downloadsinglepageblob
7574
testScenario: downloadsinglepageblob
75+
imageBuildDir: ../../../..
7676
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
7777
- image: Dockerfile
78-
imageBuildDir: ../../../..
7978
Scenario: downloaddirectorypageblob
8079
testScenario: downloaddirectorypageblob
80+
imageBuildDir: ../../../..
8181
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
8282
- image: Dockerfile
83-
imageBuildDir: ../../../..
8483
Scenario: copysinglepageblob
8584
testScenario: copysinglepageblob
85+
imageBuildDir: ../../../..
8686
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
8787
- image: Dockerfile
88-
imageBuildDir: ../../../..
8988
Scenario: copydirectorypageblob
9089
testScenario: copydirectorypageblob
90+
imageBuildDir: ../../../..
9191
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
9292

‎sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Azure.Storage.DataMovement.Blobs.Stress.csproj‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,9 @@
2121
</ItemGroup>
2222
<ItemGroup>
2323
<ProjectReference Include="$(MSBuildThisFileDirectory)..\..\..\Azure.Storage.DataMovement\src\Azure.Storage.DataMovement.csproj" />
24-
<ProjectReference Include="$(MSBuildThisFileDirectory)..\..\..\Azure.Storage.Blobs\src\Azure.Storage.Blobs.csproj" />
24+
<ProjectReference Include="$(MSBuildThisFileDirectory)..\..\..\Azure.Storage.Blobs\src\Azure.Storage.Blobs.csproj" >
25+
<Aliases>BaseBlobs</Aliases>
26+
</ProjectReference>
2527
<ProjectReference Include="$(MSBuildThisFileDirectory)..\..\src\Azure.Storage.DataMovement.Blobs.csproj" />
2628
</ItemGroup>
2729
<ItemGroup>
@@ -37,6 +39,8 @@
3739
<Compile Include="$(AzureStorageSharedSources)WindowStream.cs" LinkBase="Shared\Storage" />
3840
</ItemGroup>
3941
<ItemGroup>
42+
43+
<Compile Include="$(AzureStorageSharedTestSources)IDisposingContainer.cs" LinkBase="SharedSource\Storage" />
4044
<Compile Include="$(AzureStorageSharedTestSources)TemporaryFileStream.cs" LinkBase="SharedSource\Storage" />
4145
</ItemGroup>
4246
<ItemGroup>
@@ -46,15 +50,18 @@
4650
<Compile Include="$(AzureStorageDataMovementTestSharedSources)DisposingLocalDirectory.cs" LinkBase="Shared\DataMovement" />
4751
<Compile Include="$(AzureStorageDataMovementTestSharedSources)TransferValidator.cs" LinkBase="Shared\DataMovement" />
4852
<Compile Include="$(AzureStorageDataMovementTestSharedSources)TransferValidator.Local.cs" LinkBase="Shared\DataMovement" />
49-
<Compile Include="$(AzureStorageDataMovementTestSharedSources)../TransferValidator.Blobs.cs" LinkBase="Shared\DataMovement" />
53+
<Compile Include="$(MSBuildThisFileDirectory)..\..\..\Azure.Storage.DataMovement.Blobs\tests\DisposingBlobContainer.cs" LinkBase="Shared\DataMovement" />
54+
<Compile Include="$(MSBuildThisFileDirectory)..\..\..\Azure.Storage.DataMovement.Blobs\tests\TransferValidator.Blobs.cs" LinkBase="Shared\DataMovement" />
5055
<Compile Include="$(AzureStorageDataMovementTestSharedSources)TestEventsRaised.cs" LinkBase="Shared\DataMovement" />
5156
</ItemGroup>
5257
<ItemGroup>
5358
<Compile Include="$(AzureStorageStressTestSharedSources)*.cs" LinkBase="Shared\Stress" />
59+
<Compile Include="$(AzureStorageDataMovementTestSharedSources)..\..\stress\*.cs" LinkBase="Shared\Stress" />
5460
</ItemGroup>
5561
<ItemGroup>
5662
<Content Include="Resources\**">
5763
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
5864
</Content>
5965
</ItemGroup>
6066
</Project>
67+

‎sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Configurations/BlockBlobConfiguration.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
// Copyright (c) Microsoft Corporation. All rights reserved.
22
// Licensed under the MIT License.
3+
extern alias BaseBlobs;
34

4-
using Azure.Storage.Blobs;
5+
using BaseBlobs::Azure.Storage.Blobs;
56

67
namespace Azure.Storage.DataMovement.Blobs.Stress
78
{

‎sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Infrastructure/BlobTestSetupHelper.cs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,17 @@
11
// Copyright (c) Microsoft Corporation. All rights reserved.
22
// Licensed under the MIT License.
33

4+
extern alias BaseBlobs;
5+
46
using System;
57
using System.IO;
68
using System.Threading;
79
using System.Threading.Tasks;
810
using Azure.Storage.Shared;
9-
using Azure.Storage.Blobs;
10-
using Azure.Storage.Blobs.Specialized;
11+
using BaseBlobs::Azure.Storage.Blobs;
12+
using BaseBlobs::Azure.Storage.Blobs.Specialized;
1113
using NUnit.Framework;
12-
using Azure.Storage.Blobs.Models;
14+
using BaseBlobs::Azure.Storage.Blobs.Models;
1315

1416
namespace Azure.Storage.DataMovement.Blobs.Stress
1517
{

‎sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/BlobScenarioBase.cs‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,27 @@
11
// Copyright (c) Microsoft Corporation. All rights reserved.
22
// Licensed under the MIT License.
33

4+
extern alias BaseBlobs;
5+
46
using System;
57
using Azure.Core;
68
using Azure.Storage.Stress;
7-
using Azure.Storage.Blobs;
9+
using BaseBlobs::Azure.Storage.Blobs;
810
using System.Threading.Tasks;
911
using System.IO;
1012
using System.Threading;
13+
using Azure.Storage.DataMovement.Stress;
1114

1215
namespace Azure.Storage.DataMovement.Blobs.Stress
1316
{
14-
public abstract class BlobScenarioBase : TestScenarioBase
17+
public abstract class BlobScenarioBase : DataMovementScenarioBase
1518
{
1619
protected internal readonly Uri _destinationBlobUri;
1720
protected internal int _blobSize;
1821
protected internal readonly TokenCredential _tokenCredential;
1922
protected internal BlobsStorageResourceProvider _blobsStorageResourceProvider;
2023
protected internal LocalFilesStorageResourceProvider _localFilesStorageResourceProvider;
2124
protected internal BlobServiceClient _blobServiceClient;
22-
protected internal readonly TransferManagerOptions _transferManagerOptions;
23-
protected internal readonly DataTransferOptions _dataTransferOptions;
2425

2526
public BlobScenarioBase(
2627
Uri blobUri,
@@ -30,12 +31,10 @@ public BlobScenarioBase(
3031
TokenCredential tokenCredential,
3132
Metrics metrics,
3233
string testRunId)
33-
: base(metrics, testRunId)
34+
: base(transferManagerOptions, dataTransferOptions, metrics, testRunId)
3435
{
3536
_destinationBlobUri = blobUri;
3637
_blobSize = blobSize != default ? blobSize.Value : DataMovementBlobStressConstants.DefaultObjectSize;
37-
_transferManagerOptions = transferManagerOptions;
38-
_dataTransferOptions = dataTransferOptions;
3938
_tokenCredential = tokenCredential;
4039
_blobsStorageResourceProvider = new BlobsStorageResourceProvider(tokenCredential);
4140
_localFilesStorageResourceProvider = new LocalFilesStorageResourceProvider();

‎sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyAppendBlobDirectoryScenario.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
// Copyright (c) Microsoft Corporation. All rights reserved.
22
// Licensed under the MIT License.
3+
extern alias BaseBlobs;
34

45
using System;
56
using System.Threading;
67
using System.Threading.Tasks;
78
using Azure.Core;
8-
using Azure.Storage.Blobs.Models;
9+
using BaseBlobs::Azure.Storage.Blobs.Models;
910
using Azure.Storage.Stress;
1011

1112
namespace Azure.Storage.DataMovement.Blobs.Stress

‎sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/CopyAppendBlobSingleScenario.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
// Copyright (c) Microsoft Corporation. All rights reserved.
22
// Licensed under the MIT License.
33

4+
extern alias BaseBlobs;
5+
46
using System;
57
using System.Threading;
68
using System.Threading.Tasks;
79
using Azure.Core;
8-
using Azure.Storage.Blobs.Models;
10+
using BaseBlobs::Azure.Storage.Blobs.Models;
911
using Azure.Storage.Stress;
1012

1113
namespace Azure.Storage.DataMovement.Blobs.Stress

0 commit comments

Comments
 (0)