diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/samples/Sample01b_HelloWorldAsync.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/samples/Sample01b_HelloWorldAsync.cs index 311459026f60..36f93c2c4b23 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/samples/Sample01b_HelloWorldAsync.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/samples/Sample01b_HelloWorldAsync.cs @@ -480,7 +480,7 @@ public async Task UploadDirectory_EventHandler_SasAsync() DataTransferOptions transferOptions = new DataTransferOptions(); transferOptions.TransferStatusChanged += (TransferStatusEventArgs args) => { - if (args.StorageTransferStatus == DataTransferStatus.Completed) + if (args.TransferStatus.HasCompletedSuccessfully) { using (StreamWriter logStream = File.AppendText(logFile)) { @@ -672,7 +672,7 @@ public async Task CopySingle_ConnectionStringAsync() #endregion Assert.IsTrue(await destinationAppendBlobClient.ExistsAsync()); - Assert.AreEqual(dataTransfer.TransferStatus, DataTransferStatus.Completed); + Assert.AreEqual(DataTransferState.Completed, dataTransfer.TransferStatus.State); } finally { diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/Azure.Storage.DataMovement.Blobs.csproj b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/Azure.Storage.DataMovement.Blobs.csproj index a86fbfb8a1b3..38b9b2b134b7 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/Azure.Storage.DataMovement.Blobs.csproj +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/Azure.Storage.DataMovement.Blobs.csproj @@ -46,6 +46,7 @@ + @@ -53,7 +54,6 @@ - diff --git a/sdk/storage/Azure.Storage.DataMovement/CHANGELOG.md b/sdk/storage/Azure.Storage.DataMovement/CHANGELOG.md index 4b5a98f16d06..7130097a95c0 100644 --- a/sdk/storage/Azure.Storage.DataMovement/CHANGELOG.md +++ b/sdk/storage/Azure.Storage.DataMovement/CHANGELOG.md @@ -40,6 +40,7 @@ - [BREAKING CHANGE] Renamed `SingleTransferCompletedEventArgs` to `TransferItemCompletedEventArgs` - [BREAKING CHANGE] Renamed `TransferItemFailedEventArgs` to `TransferItemFailedEventArgs` - [BREAKING CHANGE] Renamed `TransferItemSkippedEventArgs` to `TransferItemSkippedEventArgs` +- [BREAKING CHANGE] Renamed `TransferStatusEventArgs.StorageTransferStatus` to `TransferStatus` - [BREAKING CHANGE] Renamed `StorageResourceSingle` to `StorageResourceItem` - [BREAKING CHANGE] Renamed `StorageResourceItem.WriteFromStreamAsync` to `CopyFromStreamAsync` - [BREAKING CHANGE] Renamed `StorageResourceContainer.GetChildStorageResource` to `StorageResourceContainer.GetStorageResourceReference` @@ -49,8 +50,9 @@ - [BREAKING CHANGE] Renamed `ErrorHandlingBehavior` to `DataTransferErrorMode` - [BREAKING CHANGE] Renamed `DataTransferErrorMode.StopOnAnyFailures` to `StopOnAnyFailure` - [BREAKING CHANGE] Renamed `TransferType` to `DataTransferOrder` -- [BREAKING CHANGE] Renamed `DataTransferOrder.Unordered` to `Unordered` +- [BREAKING CHANGE] Renamed `DataTransferOrder.Concurrent` to `Unordered` - [BREAKING CHANGE] Renamed `StorageTransferStatus` to `DataTransferStatus` +- [BREAKING CHANGE] Changed `DataTransferStatus` from `enum` to a `class`. - [BREAKING CHANGE] Renamed `StorageResourceCreateMode` to `StorageResourceCreationPreference`. - [BREAKING CHANGE] Renamed `StorageResourceCreationPreference` values from `Fail` to `FailIfExists`, `Overwrite` to `OverwriteIfExists` and `Skip` to `SkipIfExists`. `None` was removed, use `FailIfExists` instead. - [BREAKING CHANGE] Renamed `DataTransferOptions.CreateMode` to `CreationPreference`. diff --git a/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.net6.0.cs b/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.net6.0.cs index a41116d0d9a4..37d3a155f090 100644 --- a/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.net6.0.cs +++ b/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.net6.0.cs @@ -70,17 +70,25 @@ protected internal DataTransferProperties() { } public virtual string SourceTypeId { get { throw null; } } public virtual string TransferId { get { throw null; } } } - public enum DataTransferStatus + public enum DataTransferState { None = 0, Queued = 1, InProgress = 2, - Paused = 3, - Completed = 4, - CompletedWithSkippedTransfers = 5, - CompletedWithFailedTransfers = 6, - PauseInProgress = 7, - CancellationInProgress = 8, + Pausing = 3, + Stopping = 4, + Paused = 5, + Completed = 6, + } + public partial class DataTransferStatus : System.IEquatable + { + protected internal DataTransferStatus() { } + protected internal DataTransferStatus(Azure.Storage.DataMovement.DataTransferState state, bool hasFailureItems, bool hasSkippedItems) { } + public bool HasCompletedSuccessfully { get { throw null; } } + public bool HasFailedItems { get { throw null; } } + public bool HasSkippedItems { get { throw null; } } + public Azure.Storage.DataMovement.DataTransferState State { get { throw null; } } + public bool Equals(Azure.Storage.DataMovement.DataTransferStatus other) { throw null; } } public partial class LocalFilesStorageResourceProvider : Azure.Storage.DataMovement.StorageResourceProvider { @@ -213,6 +221,6 @@ public TransferManagerOptions() { } public partial class TransferStatusEventArgs : Azure.Storage.DataMovement.DataTransferEventArgs { public TransferStatusEventArgs(string transferId, Azure.Storage.DataMovement.DataTransferStatus transferStatus, bool isRunningSynchronously, System.Threading.CancellationToken cancellationToken) : base (default(string), default(bool), default(System.Threading.CancellationToken)) { } - public Azure.Storage.DataMovement.DataTransferStatus StorageTransferStatus { get { throw null; } } + public Azure.Storage.DataMovement.DataTransferStatus TransferStatus { get { throw null; } } } } diff --git a/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.netstandard2.0.cs b/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.netstandard2.0.cs index a41116d0d9a4..37d3a155f090 100644 --- a/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.netstandard2.0.cs @@ -70,17 +70,25 @@ protected internal DataTransferProperties() { } public virtual string SourceTypeId { get { throw null; } } public virtual string TransferId { get { throw null; } } } - public enum DataTransferStatus + public enum DataTransferState { None = 0, Queued = 1, InProgress = 2, - Paused = 3, - Completed = 4, - CompletedWithSkippedTransfers = 5, - CompletedWithFailedTransfers = 6, - PauseInProgress = 7, - CancellationInProgress = 8, + Pausing = 3, + Stopping = 4, + Paused = 5, + Completed = 6, + } + public partial class DataTransferStatus : System.IEquatable + { + protected internal DataTransferStatus() { } + protected internal DataTransferStatus(Azure.Storage.DataMovement.DataTransferState state, bool hasFailureItems, bool hasSkippedItems) { } + public bool HasCompletedSuccessfully { get { throw null; } } + public bool HasFailedItems { get { throw null; } } + public bool HasSkippedItems { get { throw null; } } + public Azure.Storage.DataMovement.DataTransferState State { get { throw null; } } + public bool Equals(Azure.Storage.DataMovement.DataTransferStatus other) { throw null; } } public partial class LocalFilesStorageResourceProvider : Azure.Storage.DataMovement.StorageResourceProvider { @@ -213,6 +221,6 @@ public TransferManagerOptions() { } public partial class TransferStatusEventArgs : Azure.Storage.DataMovement.DataTransferEventArgs { public TransferStatusEventArgs(string transferId, Azure.Storage.DataMovement.DataTransferStatus transferStatus, bool isRunningSynchronously, System.Threading.CancellationToken cancellationToken) : base (default(string), default(bool), default(System.Threading.CancellationToken)) { } - public Azure.Storage.DataMovement.DataTransferStatus StorageTransferStatus { get { throw null; } } + public Azure.Storage.DataMovement.DataTransferStatus TransferStatus { get { throw null; } } } } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/CheckpointerExtensions.cs b/sdk/storage/Azure.Storage.DataMovement/src/CheckpointerExtensions.cs index 6afab1673e4b..75ebf0a1c658 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/CheckpointerExtensions.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/CheckpointerExtensions.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +using System; using System.Threading; using System.Threading.Tasks; @@ -13,13 +14,25 @@ internal static async Task IsResumableAsync( string transferId, CancellationToken cancellationToken) { - DataTransferStatus jobStatus = (DataTransferStatus)await checkpointer.GetByteValue( + DataTransferState transferState = (DataTransferState) await checkpointer.GetByteValue( transferId, - DataMovementConstants.JobPartPlanFile.AtomicJobStatusIndex, + DataMovementConstants.JobPartPlanFile.AtomicJobStatusStateIndex, cancellationToken).ConfigureAwait(false); + byte hasFailedItemsByte = await checkpointer.GetByteValue( + transferId, + DataMovementConstants.JobPartPlanFile.AtomicJobStatusHasFailedIndex, + cancellationToken).ConfigureAwait(false); + bool hasFailedItems = Convert.ToBoolean(hasFailedItemsByte); + + byte hasSkippedItemsByte = await checkpointer.GetByteValue( + transferId, + DataMovementConstants.JobPartPlanFile.AtomicJobStatusHasSkippedIndex, + cancellationToken).ConfigureAwait(false); + bool hasSkippedItems = Convert.ToBoolean(hasSkippedItemsByte); + // Transfers marked as fully completed are not resumable - return jobStatus != DataTransferStatus.Completed; + return transferState != DataTransferState.Completed || hasFailedItems || hasSkippedItems; } internal static async Task GetDataTransferPropertiesAsync( diff --git a/sdk/storage/Azure.Storage.DataMovement/src/DataTransferState.cs b/sdk/storage/Azure.Storage.DataMovement/src/DataTransferState.cs new file mode 100644 index 000000000000..fb68ba45ce3e --- /dev/null +++ b/sdk/storage/Azure.Storage.DataMovement/src/DataTransferState.cs @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.Threading; + +namespace Azure.Storage.DataMovement +{ + /// + /// Defines the types of the state a transfer can have. + /// + public enum DataTransferState + { + /// + /// Default value. + /// + None = 0, + + /// + /// The transfer has been queued up but has not yet started. + /// + Queued = 1, + + /// + /// The transfer has started, but has not yet completed. + /// + InProgress = 2, + + /// + /// The transfer is in progress and is in the process of being paused. + /// + /// Transfer can be stopped if + /// or is called. + /// + Pausing = 3, + + /// + /// The transfer is in progress and is in the process of being stopped. + /// + /// Transfer can be stopped if is + /// enabled in the . + /// + Stopping = 4, + + /// + /// The transfer has been paused. When transfer is paused + /// (e.g. see ) + /// during the transfer, this will be the value. + /// + Paused = 5, + + /// + /// The transfer has come to a completed state. If the transfer has started and + /// has fully stopped will also come to this state. + /// + Completed = 6 + } +} diff --git a/sdk/storage/Azure.Storage.DataMovement/src/DataTransferStatus.cs b/sdk/storage/Azure.Storage.DataMovement/src/DataTransferStatus.cs index 0a56b039e54e..fde3d6e26603 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/DataTransferStatus.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/DataTransferStatus.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +using System; +using System.Runtime.CompilerServices; using System.Threading; namespace Azure.Storage.DataMovement @@ -8,59 +10,121 @@ namespace Azure.Storage.DataMovement /// /// Defines the status of the Transfer Job. /// - public enum DataTransferStatus + public class DataTransferStatus : IEquatable { + private int _hasFailedItemValue; + private int _hasSkippedItemValue; + private int _stateValue; + /// - /// Default value. - /// Equivalent to . + /// Defines the state of the transfer. /// - None = 0, + public DataTransferState State => (DataTransferState)_stateValue; /// - /// The Job has been queued up but has not yet begun any transfers. - /// Equivalent to . + /// Represents if the transfer has completed successfully without any failure or skipped items. /// - Queued = 1, + public bool HasCompletedSuccessfully => + (State == DataTransferState.Completed) && + !HasFailedItems && + !HasSkippedItems; /// - /// The Job has started, but has not yet completed. - /// Equivalent to . + /// Represents if transfer has any failure items. + /// + /// If set to `true`, the transfer has at least one failure item. + /// If set to `false`, the transfer currently has no failures. /// - InProgress = 2, + public bool HasFailedItems => _hasFailedItemValue != 0; /// - /// The Job has been paused. When transfer is paused (e.g. see ) during the transfer, - /// this will be the value. + /// Represents if transfer has any skipped items. /// - /// This status is a resumable state, only - /// transfers that failed will be retried when - /// with the respective transfer ID to resume. + /// If set to `true`, the transfer has at least one item it has skipped. + /// If set to `false`, the transfer currently has no items that has been skipped. + /// + /// It's possible to never have any items skipped if + /// is not enabled in the . /// - Paused = 3, + public bool HasSkippedItems => _hasSkippedItemValue != 0; /// - /// The Job has completed successfully with no failures or skips. + /// Constructor to set the initial state to with no failures or skipped items. /// - Completed = 4, + protected internal DataTransferStatus() + { + _stateValue = (int)DataTransferState.Queued; + _hasFailedItemValue = 0; // Initialized to false + _hasSkippedItemValue = 0; // Initialized to false + } /// - /// The Job has been completed with at least one skipped transfer. + /// Constructor to have a custom state, failure state, and skipped state. /// - CompletedWithSkippedTransfers = 5, + protected internal DataTransferStatus(DataTransferState state, bool hasFailureItems, bool hasSkippedItems) + { + _stateValue = (int)state; + _hasFailedItemValue = hasFailureItems ? 1 : 0; + _hasSkippedItemValue = hasSkippedItems ? 1 : 0; + } + + internal bool IsCompletedWithFailedItems => State.Equals(DataTransferState.Completed) && HasFailedItems; + internal bool IsCompletedWithSkippedItems => State.Equals(DataTransferState.Completed) && HasSkippedItems; /// - /// The Job has been completed with at least one failed transfer. + /// Accordingly update the to true. If already set to true, nothing will happen. + /// + /// This should only be triggered when a failed item has been seen. + /// + /// True if was updated. False otherwise. + internal bool TrySetFailedItem() + { + return Interlocked.Exchange(ref _hasFailedItemValue, 1) != 1; + } + + /// + /// Accordingly update the to true. If already set to true, nothing will happen. + /// + /// This should only be triggered when a skipped item has been seen. + /// + /// /// True if was updated. False otherwise. + internal bool TrySetSkippedItem() + { + return Interlocked.Exchange(ref _hasSkippedItemValue, 1) != 1; + } + + /// + /// Accordingly update the . If the current State is the same as the parameter, + /// then nothing will happen. + /// + /// This should only be triggered when the state updates. /// - CompletedWithFailedTransfers = 6, + /// True if was updated. False otherwise. + internal bool TrySetTransferStateChange(DataTransferState state) + { + return Interlocked.Exchange(ref _stateValue, (int)state) != (int)state; + } /// - /// A pause was called on the transfer job and is in progress. + /// Indicates whether the current object is equal to another object of the same type. /// - PauseInProgress = 7, + /// An object to compare with this object. + /// Returns true if the current object is equal to the other parameter; otherwise, false. + public bool Equals(DataTransferStatus other) + => State.Equals(other.State) && + HasFailedItems.Equals(other.HasFailedItems) && + HasSkippedItems.Equals(other.HasSkippedItems); /// - /// A pause was called on the transfer job and is in progress. + /// Performs a Deep Copy of the . /// - CancellationInProgress = 8, - }; + /// A deep copy of the respective . + internal DataTransferStatus DeepCopy() + => new() + { + _stateValue = _stateValue, + _hasFailedItemValue = _hasFailedItemValue, + _hasSkippedItemValue = _hasSkippedItemValue, + }; + } } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/JobPartInternal.cs b/sdk/storage/Azure.Storage.DataMovement/src/JobPartInternal.cs index 92e532488fad..d51e7d7a438d 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/JobPartInternal.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/JobPartInternal.cs @@ -96,7 +96,6 @@ internal abstract class JobPartInternal /// The current status of each job part. /// public DataTransferStatus JobPartStatus { get; set; } - private object _statusLock = new object(); /// /// Optional. If the length is known, we log it instead of doing a GetProperties call on the @@ -169,12 +168,13 @@ internal JobPartInternal( SyncAsyncEventHandler singleTransferEventHandler, ClientDiagnostics clientDiagnostics, CancellationToken cancellationToken, - DataTransferStatus jobPartStatus = DataTransferStatus.Queued, + DataTransferStatus jobPartStatus = default, long? length = default) { Argument.AssertNotNull(clientDiagnostics, nameof(clientDiagnostics)); - JobPartStatus = jobPartStatus; + // if defualt is passed, the job part status will be queued + JobPartStatus = jobPartStatus ?? new DataTransferStatus(); PartNumber = partNumber; _dataTransfer = dataTransfer; _sourceResource = sourceResource; @@ -241,7 +241,7 @@ await QueueChunk( { await Task.Run(chunkTask).ConfigureAwait(false); chunkCompleted.SetResult(true); - await CheckAndUpdateCancellationStatusAsync().ConfigureAwait(false); + await CheckAndUpdateCancellationStateAsync().ConfigureAwait(false); } catch (Exception ex) { @@ -259,28 +259,29 @@ await QueueChunk( /// /// Triggers the cancellation for the Job Part. /// - /// If the status is set to - /// and any chunks is still processing to be cancelled is will be set to - /// until the chunks finish then it will be set to . + /// If the status is set to + /// and any chunks is still processing to be cancelled is will be set to + /// until the chunks finish then it will be set to . /// - /// If the status is set to - /// and any chunks is still processing to be cancelled is will be set to - /// until the chunks finish then it will be set to . + /// If the part status is set to but has + /// + /// and any chunks is still processing to be cancelled is will be set to + /// until the chunks finish then it will be set to . /// /// The task to wait until the cancellation has been triggered. internal async Task TriggerCancellationAsync() { // Set the status to Pause/CancellationInProgress - if (DataTransferStatus.PauseInProgress == _dataTransfer.TransferStatus) + if (DataTransferState.Pausing == _dataTransfer.TransferStatus.State) { // It's possible that the status hasn't propagated down to the job part // status yet here since we pause from the data transfer object. - await OnTransferStatusChanged(DataTransferStatus.PauseInProgress).ConfigureAwait(false); + await OnTransferStateChangedAsync(DataTransferState.Pausing).ConfigureAwait(false); } else { // It's a cancellation if a pause wasn't called. - await OnTransferStatusChanged(DataTransferStatus.CancellationInProgress).ConfigureAwait(false); + await OnTransferStateChangedAsync(DataTransferState.Stopping).ConfigureAwait(false); } await CleanupAbortedJobPartAsync().ConfigureAwait(false); } @@ -288,52 +289,29 @@ internal async Task TriggerCancellationAsync() /// /// To change all transfer statues at the same time /// - /// - internal async Task OnTransferStatusChanged(DataTransferStatus transferStatus) + /// + internal async Task OnTransferStateChangedAsync(DataTransferState transferState) { - bool statusChanged = false; - lock (_statusLock) - { - if (transferStatus != DataTransferStatus.None - && JobPartStatus != transferStatus) - { - statusChanged = true; - JobPartStatus = transferStatus; - } - } - if (statusChanged) + if (JobPartStatus.TrySetTransferStateChange(transferState)) { // Progress tracking, do before invoking the event below - if (transferStatus == DataTransferStatus.InProgress) + if (transferState == DataTransferState.InProgress) { _progressTracker.IncrementInProgressFiles(); } - else if (transferStatus == DataTransferStatus.Completed) + else if (JobPartStatus.HasCompletedSuccessfully) { _progressTracker.IncrementCompletedFiles(); - } - else if (transferStatus == DataTransferStatus.CompletedWithSkippedTransfers) - { - _progressTracker.IncrementSkippedFiles(); - } - else if (transferStatus == DataTransferStatus.CompletedWithFailedTransfers) - { - _progressTracker.IncrementFailedFiles(); - } - - if (JobPartStatus == DataTransferStatus.Completed) - { await InvokeSingleCompletedArg().ConfigureAwait(false); } // Set the status in the checkpointer - await SetCheckpointerStatus(transferStatus).ConfigureAwait(false); + await SetCheckpointerStatus().ConfigureAwait(false); - // TODO: change to RaiseAsync await PartTransferStatusEventHandler.RaiseAsync( new TransferStatusEventArgs( _dataTransfer.Id, - transferStatus, + JobPartStatus, false, _cancellationToken), nameof(JobPartInternal), @@ -390,7 +368,25 @@ await TransferSkippedEventHandler.RaiseAsync( ClientDiagnostics) .ConfigureAwait(false); } - await OnTransferStatusChanged(DataTransferStatus.CompletedWithSkippedTransfers).ConfigureAwait(false); + _progressTracker.IncrementSkippedFiles(); + + // Update the JobPartStatus. If was already updated (e.g. there was a failed item before) + // then don't raise the PartTransferStatusEventHandler + if (JobPartStatus.TrySetSkippedItem()) + { + await PartTransferStatusEventHandler.RaiseAsync( + new TransferStatusEventArgs( + _dataTransfer.Id, + JobPartStatus, + false, + _cancellationToken), + nameof(JobPartInternal), + nameof(PartTransferStatusEventHandler), + ClientDiagnostics) + .ConfigureAwait(false); + } + //TODO: figure out why we set the Completed state here and not just wait for all the chunks to finish + await OnTransferStateChangedAsync(DataTransferState.Completed).ConfigureAwait(false); } /// @@ -418,10 +414,27 @@ await TransferFailedEventHandler.RaiseAsync( ClientDiagnostics) .ConfigureAwait(false); } + _progressTracker.IncrementFailedFiles(); + + // Update the JobPartStatus. If was already updated (e.g. there was a failed item before) + // then don't raise the PartTransferStatusEventHandler + if (JobPartStatus.TrySetFailedItem()) + { + await PartTransferStatusEventHandler.RaiseAsync( + new TransferStatusEventArgs( + _dataTransfer.Id, + JobPartStatus, + false, + _cancellationToken), + nameof(JobPartInternal), + nameof(PartTransferStatusEventHandler), + ClientDiagnostics) + .ConfigureAwait(false); + } } // Trigger job cancellation if the failed handler is enabled await TriggerCancellationAsync().ConfigureAwait(false); - await CheckAndUpdateCancellationStatusAsync().ConfigureAwait(false); + await CheckAndUpdateCancellationStateAsync().ConfigureAwait(false); } /// @@ -452,7 +465,7 @@ public async virtual Task CleanupAbortedJobPartAsync() public async virtual Task AddJobPartToCheckpointerAsync(int chunksTotal, bool isFinalPart) { JobPartPlanHeader header = this.ToJobPartPlanHeader( - jobStatus: DataTransferStatus.InProgress, + jobStatus: JobPartStatus, isFinalPart: isFinalPart); using (Stream stream = new MemoryStream()) { @@ -466,12 +479,12 @@ await _checkpointer.AddNewJobPartAsync( } } - internal async virtual Task SetCheckpointerStatus(DataTransferStatus status) + internal async virtual Task SetCheckpointerStatus() { await _checkpointer.SetJobPartTransferStatusAsync( transferId: _dataTransfer.Id, partNumber: PartNumber, - status: status).ConfigureAwait(false); + status: JobPartStatus).ConfigureAwait(false); } internal long CalculateBlockSize(long length) @@ -563,17 +576,17 @@ internal static long ParseRangeTotalLength(string range) } } - internal async Task CheckAndUpdateCancellationStatusAsync() + internal async Task CheckAndUpdateCancellationStateAsync() { if (_chunkTasks.All((Task task) => (task.IsCompleted))) { - if (JobPartStatus == DataTransferStatus.PauseInProgress) + if (JobPartStatus.State == DataTransferState.Pausing) { - await OnTransferStatusChanged(DataTransferStatus.Paused).ConfigureAwait(false); + await OnTransferStateChangedAsync(DataTransferState.Paused).ConfigureAwait(false); } - else if (JobPartStatus == DataTransferStatus.CancellationInProgress) + else if (JobPartStatus.State == DataTransferState.Stopping) { - await OnTransferStatusChanged(DataTransferStatus.CompletedWithFailedTransfers).ConfigureAwait(false); + await OnTransferStateChangedAsync(DataTransferState.Completed).ConfigureAwait(false); } } } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceJobPart.cs b/sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceJobPart.cs index 277a045fe6e3..f0fce0e2c19f 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceJobPart.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceJobPart.cs @@ -55,7 +55,7 @@ private ServiceToServiceJobPart( StorageResourceItem sourceResource, StorageResourceItem destinationResource, bool isFinalPart, - DataTransferStatus jobPartStatus = DataTransferStatus.Queued, + DataTransferStatus jobPartStatus = default, long? length = default) : base(dataTransfer: job._dataTransfer, partNumber: partNumber, @@ -128,7 +128,7 @@ public static async Task CreateJobPartAsync( public override async Task ProcessPartToChunkAsync() { - await OnTransferStatusChanged(DataTransferStatus.InProgress).ConfigureAwait(false); + await OnTransferStateChangedAsync(DataTransferState.InProgress).ConfigureAwait(false); // Attempt to get the length, it's possible the file could // not be accessible (or does not exist). @@ -193,7 +193,7 @@ await PutBlockFromUri( } else { - await CheckAndUpdateCancellationStatusAsync().ConfigureAwait(false); + await CheckAndUpdateCancellationStateAsync().ConfigureAwait(false); } } @@ -211,7 +211,7 @@ await _destinationResource.CopyFromUriAsync( cancellationToken: _cancellationToken).ConfigureAwait(false); ReportBytesWritten(completeLength); - await OnTransferStatusChanged(DataTransferStatus.Completed).ConfigureAwait(false); + await OnTransferStateChangedAsync(DataTransferState.Completed).ConfigureAwait(false); } catch (RequestFailedException exception) when (_createMode == StorageResourceCreationPreference.SkipIfExists @@ -307,7 +307,7 @@ await _destinationResource.CompleteTransferAsync( await DisposeHandlers().ConfigureAwait(false); // Set completion status to completed - await OnTransferStatusChanged(DataTransferStatus.Completed).ConfigureAwait(false); + await OnTransferStateChangedAsync(DataTransferState.Completed).ConfigureAwait(false); } catch (Exception ex) { diff --git a/sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceTransferJob.cs b/sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceTransferJob.cs index 637076852ac4..ee6217f4b7e7 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceTransferJob.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceTransferJob.cs @@ -69,7 +69,7 @@ internal ServiceToServiceTransferJob( /// An IEnumerable that contains the job parts public override async IAsyncEnumerable ProcessJobToJobPartAsync() { - await OnJobStatusChangedAsync(DataTransferStatus.InProgress).ConfigureAwait(false); + await OnJobStateChangedAsync(DataTransferState.InProgress).ConfigureAwait(false); int partNumber = 0; if (_jobParts.Count == 0) @@ -108,9 +108,9 @@ public override async IAsyncEnumerable ProcessJobToJobPartAsync bool isFinalPartFound = false; foreach (JobPartInternal part in _jobParts) { - if (part.JobPartStatus != DataTransferStatus.Completed) + if (!part.JobPartStatus.HasCompletedSuccessfully) { - part.JobPartStatus = DataTransferStatus.Queued; + part.JobPartStatus.TrySetTransferStateChange(DataTransferState.Queued); yield return part; if (part.IsFinalPart) diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataMovementConstants.cs b/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataMovementConstants.cs index 1dda7bad9daa..d5eda17999c7 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataMovementConstants.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataMovementConstants.cs @@ -92,7 +92,8 @@ internal static class JobPartPlanFile { internal const string SchemaVersion_b1 = "b1"; internal const string SchemaVersion_b2 = "b2"; - internal const string SchemaVersion = SchemaVersion_b2; // TODO: remove b for beta + internal const string SchemaVersion_b3 = "b3"; + internal const string SchemaVersion = SchemaVersion_b3; // TODO: remove b for beta // Job Plan file extension. e.g. the file extension will look like {transferid}--{jobpartNumber}.steV{schemaVersion} internal const string FileExtension = ".steV"; @@ -244,13 +245,21 @@ internal static class JobPartPlanFile /// Index: 49021 internal const int RehydratePriorityTypeIndex = PermanentDeleteOptionIndex + OneByte; /// Index: 49022 - internal const int AtomicJobStatusIndex = RehydratePriorityTypeIndex + OneByte; + internal const int AtomicJobStatusStateIndex = RehydratePriorityTypeIndex + OneByte; /// Index: 49023 - internal const int AtomicPartStatusIndex = AtomicJobStatusIndex + OneByte; + internal const int AtomicJobStatusHasFailedIndex = AtomicJobStatusStateIndex + OneByte; + /// Index: 49024 + internal const int AtomicJobStatusHasSkippedIndex = AtomicJobStatusHasFailedIndex + OneByte; + /// Index: 49025 + internal const int AtomicPartStatusStateIndex = AtomicJobStatusHasSkippedIndex + OneByte; + /// Index: 49026 + internal const int AtomicPartStatusHasFailedIndex = AtomicPartStatusStateIndex + OneByte; + /// Index: 49027 + internal const int AtomicPartStatusHasSkippedIndex = AtomicPartStatusHasFailedIndex + OneByte; /// - /// Size of the JobPart Header: 49024 + /// Size of the JobPart Header: 49029 /// - internal const int JobPartHeaderSizeInBytes = AtomicPartStatusIndex + OneByte; + internal const int JobPartHeaderSizeInBytes = AtomicPartStatusHasSkippedIndex + OneByte; } internal static class ErrorCode diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataTransfer.cs b/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataTransfer.cs index f1f6becbe1bc..12ab5aac84e9 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataTransfer.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataTransfer.cs @@ -36,7 +36,7 @@ public class DataTransfer /// /// Defines the current state of the transfer. /// - internal DataTransferState _state; + internal DataTransferInternalState _state; /// /// For mocking. @@ -54,11 +54,12 @@ internal DataTransfer() internal DataTransfer( string id, TransferManager transferManager, - DataTransferStatus status = DataTransferStatus.Queued) + DataTransferStatus status = default) { Argument.AssertNotNullOrEmpty(id, nameof(id)); Argument.AssertNotNull(transferManager, nameof(transferManager)); - _state = new DataTransferState(id, status); + status ??= new DataTransferStatus(); + _state = new DataTransferInternalState(id, status); TransferManager = transferManager; } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataTransferState.cs b/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataTransferInternalState.cs similarity index 64% rename from sdk/storage/Azure.Storage.DataMovement/src/Shared/DataTransferState.cs rename to sdk/storage/Azure.Storage.DataMovement/src/Shared/DataTransferInternalState.cs index d96830068f36..2cdc83a106ad 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataTransferState.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataTransferInternalState.cs @@ -12,9 +12,8 @@ namespace Azure.Storage.DataMovement /// /// Defines the state of the transfer /// - internal class DataTransferState + internal class DataTransferInternalState { - private readonly object _statusLock = new object(); private string _id; private DataTransferStatus _status; @@ -29,18 +28,16 @@ internal class DataTransferState /// /// The transfer ID of the transfer object. /// The Transfer Status of the Transfer. See . - public DataTransferState( + public DataTransferInternalState( string id = default, - DataTransferStatus status = DataTransferStatus.Queued) + DataTransferStatus status = default) { _id = string.IsNullOrEmpty(id) ? Guid.NewGuid().ToString() : id; _status = status; CompletionSource = new TaskCompletionSource( _status, TaskCreationOptions.RunContinuationsAsynchronously); - if (DataTransferStatus.Completed == status || - DataTransferStatus.CompletedWithSkippedTransfers == status || - DataTransferStatus.CompletedWithFailedTransfers == status) + if (DataTransferState.Completed == status.State) { CompletionSource.TrySetResult(status); } @@ -62,9 +59,7 @@ internal set { } public bool HasCompleted { get { - return (DataTransferStatus.Completed == _status || - DataTransferStatus.CompletedWithSkippedTransfers == _status || - DataTransferStatus.CompletedWithFailedTransfers == _status); + return DataTransferState.Completed == _status.State; } internal set { } } @@ -84,42 +79,37 @@ public void SetId(string id) /// public DataTransferStatus GetTransferStatus() { - lock (_statusLock) - { - return _status; - } + return _status; } /// /// Sets the completion status /// - /// + /// /// Returns whether or not the status has been changed/set - public bool TrySetTransferStatus(DataTransferStatus status) + public bool TrySetTransferState(DataTransferState state) { - lock (_statusLock) + if (_status.TrySetTransferStateChange(state)) { - if (_status != status) + if (DataTransferState.Completed == _status.State || + DataTransferState.Paused == _status.State) { - _status = status; - if (DataTransferStatus.Paused == status || - DataTransferStatus.Completed == status || - DataTransferStatus.CompletedWithSkippedTransfers == status || - DataTransferStatus.CompletedWithFailedTransfers == status) - { - // If the _completionSource has been cancelled or the exception - // has been set, we don't need to check if TrySetResult returns false - // because it's acceptable to cancel or have an error occur before then. - CompletionSource.TrySetResult(status); - } - return true; + // If the _completionSource has been cancelled or the exception + // has been set, we don't need to check if TrySetResult returns false + // because it's acceptable to cancel or have an error occur before then. + CompletionSource.TrySetResult(_status); } - return false; + return true; } + return false; } + public bool TrySetFailedItemsState() => _status.TrySetFailedItem(); + + public bool TrySetSkippedItemsState() => _status.TrySetSkippedItem(); + internal bool CanPause() - => _status == DataTransferStatus.InProgress; + => DataTransferState.InProgress == _status.State; public async Task PauseIfRunningAsync(CancellationToken cancellationToken) { @@ -129,7 +119,7 @@ public async Task PauseIfRunningAsync(CancellationToken cancellationToken) } CancellationHelper.ThrowIfCancellationRequested(cancellationToken); // Call the inner cancellation token to stop the transfer job - TrySetTransferStatus(DataTransferStatus.PauseInProgress); + TrySetTransferState(DataTransferState.Pausing); if (TriggerCancellation()) { // Wait until full pause has completed. diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataTransferStatusInternal.cs b/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataTransferStatusInternal.cs new file mode 100644 index 000000000000..ab00ece7466e --- /dev/null +++ b/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataTransferStatusInternal.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Storage.DataMovement +{ + internal class DataTransferStatusInternal : DataTransferStatus + { + public DataTransferStatusInternal( + DataTransferState state, + bool hasFailedItems, + bool hasSkippedItems) + : base(state, hasFailedItems, hasSkippedItems) + { } + } +} diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPartPlanHeader.cs b/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPartPlanHeader.cs index 16da3241be64..b7790c40a731 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPartPlanHeader.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPartPlanHeader.cs @@ -574,11 +574,23 @@ public void Serialize(Stream stream) // RehydratePriorityType writer.Write((byte)RehydratePriorityType); - // AtomicJobStatus - writer.Write((byte)AtomicJobStatus); + // AtomicJobStatus.State + writer.Write((byte)AtomicJobStatus.State); - // AtomicPartStatus - writer.Write((byte)AtomicPartStatus); + // AtomicJobStatus.HasFailedItems + writer.Write(Convert.ToByte(AtomicJobStatus.HasFailedItems)); + + // AtomicJobStatus.HasSkippedItems + writer.Write(Convert.ToByte(AtomicJobStatus.HasSkippedItems)); + + // AtomicPartStatus.State + writer.Write((byte)AtomicPartStatus.State); + + // AtomicPartStatus.HasFailedItems + writer.Write(Convert.ToByte(AtomicPartStatus.HasFailedItems)); + + // AtomicPartStatus.HasSkippedItems + writer.Write(Convert.ToByte(AtomicPartStatus.HasSkippedItems)); } public static JobPartPlanHeader Deserialize(Stream stream) @@ -820,13 +832,35 @@ public static JobPartPlanHeader Deserialize(Stream stream) byte rehydratePriorityTypeByte = reader.ReadByte(); JobPartPlanRehydratePriorityType rehydratePriorityType = (JobPartPlanRehydratePriorityType)rehydratePriorityTypeByte; - // AtomicJobStatus + // AtomicJobStatus.State byte atomicJobStatusByte = reader.ReadByte(); - DataTransferStatus atomicJobStatus = (DataTransferStatus)atomicJobStatusByte; + DataTransferState jobStatusState = (DataTransferState)atomicJobStatusByte; + + // AtomicJobStatus.HasFailedItems + bool jobStatusHasFailed = Convert.ToBoolean(reader.ReadByte()); - // AtomicPartStatus + // AtomicJobStatus.HasSkippedItems + bool jobStatusHasSkipped = Convert.ToBoolean(reader.ReadByte()); + + // AtomicPartStatus.State byte atomicPartStatusByte = reader.ReadByte(); - DataTransferStatus atomicPartStatus = (DataTransferStatus)atomicPartStatusByte; + DataTransferState partStatusState = (DataTransferState)atomicPartStatusByte; + + // AtomicPartStatus.HasFailedItems + bool partStatusHasFailed = Convert.ToBoolean(reader.ReadByte()); + + // AtomicPartStatus.HasSkippedItems + bool partStatusHasSkipped = Convert.ToBoolean(reader.ReadByte()); + + DataTransferStatus atomicJobStatus = new DataTransferStatusInternal( + jobStatusState, + jobStatusHasFailed, + jobStatusHasSkipped); + + DataTransferStatus atomicPartStatus = new DataTransferStatusInternal( + partStatusState, + partStatusHasFailed, + partStatusHasSkipped); JobPartPlanDestinationBlob dstBlobData = new JobPartPlanDestinationBlob( blobType: blobType, diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/LocalTransferCheckpointer.cs b/sdk/storage/Azure.Storage.DataMovement/src/Shared/LocalTransferCheckpointer.cs index 58e0fb3d5107..b2f1e665e315 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/Shared/LocalTransferCheckpointer.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/Shared/LocalTransferCheckpointer.cs @@ -305,8 +305,9 @@ public override async Task SetJobTransferStatusAsync( DataTransferStatus status, CancellationToken cancellationToken = default) { - long length = DataMovementConstants.OneByte; - int offset = DataMovementConstants.JobPartPlanFile.AtomicJobStatusIndex; + long length = DataMovementConstants.OneByte * 3; + int offset = DataMovementConstants.JobPartPlanFile.AtomicJobStatusStateIndex; + CancellationHelper.ThrowIfCancellationRequested(cancellationToken); if (_transferStates.TryGetValue(transferId, out Dictionary jobPartFiles)) @@ -326,7 +327,13 @@ public override async Task SetJobTransferStatusAsync( { accessor.Write( position: 0, - value: (byte)status); + value: (byte)status.State); + accessor.Write( + position: 1, + value: status.HasFailedItems); + accessor.Write( + position: 2, + value: status.HasSkippedItems); // to flush to the underlying file that supports the mmf accessor.Flush(); } @@ -348,8 +355,9 @@ public override async Task SetJobPartTransferStatusAsync( DataTransferStatus status, CancellationToken cancellationToken = default) { - long length = DataMovementConstants.OneByte; - int offset = DataMovementConstants.JobPartPlanFile.AtomicPartStatusIndex; + long length = DataMovementConstants.OneByte * 3; + int offset = DataMovementConstants.JobPartPlanFile.AtomicPartStatusStateIndex; + CancellationHelper.ThrowIfCancellationRequested(cancellationToken); if (_transferStates.TryGetValue(transferId, out Dictionary jobPartFiles)) @@ -369,7 +377,13 @@ public override async Task SetJobPartTransferStatusAsync( { accessor.Write( position: 0, - value: (byte)status); + value: (byte)status.State); + accessor.Write( + position: 1, + value: status.HasFailedItems); + accessor.Write( + position: 2, + value: status.HasSkippedItems); // to flush to the underlying file that supports the mmf accessor.Flush(); } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/TransferManager.cs b/sdk/storage/Azure.Storage.DataMovement/src/Shared/TransferManager.cs index d4d0a5d6a31b..f6bb1ebf4635 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/Shared/TransferManager.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/Shared/TransferManager.cs @@ -260,7 +260,7 @@ public virtual async Task PauseTransferIfRunningAsync(string transferId, Cancell /// If specified, the returned list of transfers will have only have the transfers /// of which match the status specified. /// - /// If not specified or specified to , + /// If not specified or specified to , /// all transfers will be returned regardless of status. /// /// diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/TransferStatusEventArgs.cs b/sdk/storage/Azure.Storage.DataMovement/src/Shared/TransferStatusEventArgs.cs index dd534f19803f..1c6fbeded0d3 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/Shared/TransferStatusEventArgs.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/Shared/TransferStatusEventArgs.cs @@ -12,7 +12,7 @@ public class TransferStatusEventArgs : DataTransferEventArgs /// /// Gets the of the job. /// - public DataTransferStatus StorageTransferStatus { get; } + public DataTransferStatus TransferStatus { get; } /// /// Initializes a new instance of the . @@ -28,7 +28,7 @@ public TransferStatusEventArgs( CancellationToken cancellationToken) : base (transferId, isRunningSynchronously, cancellationToken) { - StorageTransferStatus = transferStatus; + TransferStatus = transferStatus; } } } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/StreamToUriJobPart.cs b/sdk/storage/Azure.Storage.DataMovement/src/StreamToUriJobPart.cs index 31dd0358f7f3..466e1fec9f6a 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/StreamToUriJobPart.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/StreamToUriJobPart.cs @@ -58,7 +58,7 @@ private StreamToUriJobPart( StorageResourceItem sourceResource, StorageResourceItem destinationResource, bool isFinalPart, - DataTransferStatus jobPartStatus = DataTransferStatus.Queued, + DataTransferStatus jobPartStatus = default, long? length = default) : base(dataTransfer: job._dataTransfer, partNumber: partNumber, @@ -142,7 +142,7 @@ public override async Task ProcessPartToChunkAsync() // Attempt to get the length, it's possible the file could // not be accesible (or does not exist). string operationName = $"{nameof(TransferManager.StartTransferAsync)}"; - await OnTransferStatusChanged(DataTransferStatus.InProgress).ConfigureAwait(false); + await OnTransferStateChangedAsync(DataTransferState.InProgress).ConfigureAwait(false); long? fileLength = default; try { @@ -250,7 +250,7 @@ await _destinationResource.CopyFromStreamAsync( ReportBytesWritten(blockSize); // Set completion status to completed - await OnTransferStatusChanged(DataTransferStatus.Completed).ConfigureAwait(false); + await OnTransferStateChangedAsync(DataTransferState.Completed).ConfigureAwait(false); } else { @@ -395,7 +395,7 @@ await _destinationResource.CompleteTransferAsync( await DisposeHandlers().ConfigureAwait(false); // Set completion status to completed - await OnTransferStatusChanged(DataTransferStatus.Completed).ConfigureAwait(false); + await OnTransferStateChangedAsync(DataTransferState.Completed).ConfigureAwait(false); } private async Task QueueStageBlockRequests(List<(long Offset, long Size)> rangeList, long completeLength) diff --git a/sdk/storage/Azure.Storage.DataMovement/src/StreamToUriTransferJob.cs b/sdk/storage/Azure.Storage.DataMovement/src/StreamToUriTransferJob.cs index 5d112ed020aa..df37661cdb91 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/StreamToUriTransferJob.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/StreamToUriTransferJob.cs @@ -67,7 +67,7 @@ internal StreamToUriTransferJob( /// An IEnumerable that contains the job parts public override async IAsyncEnumerable ProcessJobToJobPartAsync() { - await OnJobStatusChangedAsync(DataTransferStatus.InProgress).ConfigureAwait(false); + await OnJobStateChangedAsync(DataTransferState.InProgress).ConfigureAwait(false); int partNumber = 0; if (_jobParts.Count == 0) @@ -106,9 +106,9 @@ public override async IAsyncEnumerable ProcessJobToJobPartAsync bool isFinalPartFound = false; foreach (JobPartInternal part in _jobParts) { - if (part.JobPartStatus != DataTransferStatus.Completed) + if (!part.JobPartStatus.HasCompletedSuccessfully) { - part.JobPartStatus = DataTransferStatus.Queued; + part.JobPartStatus.TrySetTransferStateChange(DataTransferState.Queued); yield return part; if (part.IsFinalPart) diff --git a/sdk/storage/Azure.Storage.DataMovement/src/TransferJobInternal.cs b/sdk/storage/Azure.Storage.DataMovement/src/TransferJobInternal.cs index e35fd7e6bec9..c030881e72de 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/TransferJobInternal.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/TransferJobInternal.cs @@ -87,14 +87,6 @@ internal abstract class TransferJobInternal : IDisposable /// internal StorageResourceCreationPreference _creationPreference; - private object _statusLock = new object(); - - /// - /// To help set the job status when all job parts have completed. - /// - internal bool _hasFailures; - internal bool _hasSkipped; - /// /// Considering if there's more than one job part, the transfer status will need to be set to /// completed all job parts have been set to completed. @@ -136,8 +128,6 @@ internal abstract class TransferJobInternal : IDisposable internal bool _enumerationComplete; private int _pendingJobParts; private bool _jobPartPaused; - private bool _jobPartFailed; - private bool _jobPartSkipped; public CancellationToken _cancellationToken { get; internal set; } @@ -164,13 +154,11 @@ private TransferJobInternal( Argument.AssertNotNull(clientDiagnostics, nameof(clientDiagnostics)); _dataTransfer = dataTransfer ?? throw Errors.ArgumentNull(nameof(dataTransfer)); - _dataTransfer._state.TrySetTransferStatus(DataTransferStatus.Queued); + _dataTransfer.TransferStatus.TrySetTransferStateChange(DataTransferState.Queued); _errorMode = errorHandling; _checkpointer = checkPointer; _creationPreference = creationPreference; QueueChunkTask = queueChunkTask; - _hasFailures = false; - _hasSkipped = false; _arrayPool = arrayPool; _jobParts = new List(); _enumerationComplete = false; @@ -273,20 +261,14 @@ public void DisposeHandlers() /// /// Triggers the cancellation for the Job Part. /// - /// If the status is set to - /// and any chunks is still processing to be cancelled is will be set to - /// until the chunks finish then it will be set to . - /// - /// If the status is set to - /// and any chunks is still processing to be cancelled is will be set to - /// until the chunks finish then it will be set to . + /// If the cancellation token was called to cancelled, change the status to Stopping. /// /// The task to wait until the cancellation has been triggered. public async Task TriggerJobCancellationAsync() { if (!_dataTransfer._state.CancellationTokenSource.IsCancellationRequested) { - await OnJobStatusChangedAsync(DataTransferStatus.CancellationInProgress).ConfigureAwait(false); + await OnJobStateChangedAsync(DataTransferState.Stopping).ConfigureAwait(false); _dataTransfer._state.TriggerCancellation(); } } @@ -302,7 +284,6 @@ public async virtual Task InvokeFailedArgAsync(Exception ex) { if (TransferFailedEventHandler != null) { - // TODO: change to RaiseAsync await TransferFailedEventHandler.RaiseAsync( new TransferItemFailedEventArgs( _dataTransfer.Id, @@ -317,7 +298,6 @@ await TransferFailedEventHandler.RaiseAsync( .ConfigureAwait(false); } } - // Trigger job cancellation if the failed handler is enabled await TriggerJobCancellationAsync().ConfigureAwait(false); // If we're failing from a Transfer Job point, it means we have aborted the job @@ -332,43 +312,48 @@ await TransferFailedEventHandler.RaiseAsync( /// public async Task JobPartEvent(TransferStatusEventArgs args) { - DataTransferStatus jobPartStatus = args.StorageTransferStatus; - DataTransferStatus jobStatus = _dataTransfer._state.GetTransferStatus(); + DataTransferStatus jobPartStatus = args.TransferStatus; + DataTransferState jobState = _dataTransfer._state.GetTransferStatus().State; // Keep track of paused, failed, and skipped which we will use to determine final job status - if (jobPartStatus == DataTransferStatus.Paused) + // Since this is each Job Part coming in, the state of skipped or failed is mutually exclusive. + if (jobPartStatus.State == DataTransferState.Paused) { _jobPartPaused = true; } - else if (jobPartStatus == DataTransferStatus.CompletedWithFailedTransfers) + else if (jobPartStatus.HasFailedItems) { - _jobPartFailed = true; + if (_dataTransfer._state.TrySetFailedItemsState()) + { + await SetCheckpointerStatus().ConfigureAwait(false); + await OnJobPartStatusChangedAsync().ConfigureAwait(false); + } } - else if (jobPartStatus == DataTransferStatus.CompletedWithSkippedTransfers) + else if (jobPartStatus.HasSkippedItems) { - _jobPartSkipped = true; + if (_dataTransfer._state.TrySetSkippedItemsState()) + { + await SetCheckpointerStatus().ConfigureAwait(false); + await OnJobPartStatusChangedAsync().ConfigureAwait(false); + } } // Cancel the entire job if one job part fails and StopOnFailure is set if (_errorMode == DataTransferErrorMode.StopOnAnyFailure && - jobPartStatus == DataTransferStatus.CompletedWithFailedTransfers && - jobStatus != DataTransferStatus.CancellationInProgress && - jobStatus != DataTransferStatus.CompletedWithFailedTransfers && - jobStatus != DataTransferStatus.CompletedWithSkippedTransfers && - jobStatus != DataTransferStatus.Completed) + jobPartStatus.HasFailedItems && + jobState != DataTransferState.Stopping && + jobState != DataTransferState.Completed) { await TriggerJobCancellationAsync().ConfigureAwait(false); - jobStatus = _dataTransfer._state.GetTransferStatus(); + jobState = _dataTransfer._state.GetTransferStatus().State; } - if ((jobPartStatus == DataTransferStatus.Paused || - jobPartStatus == DataTransferStatus.Completed || - jobPartStatus == DataTransferStatus.CompletedWithSkippedTransfers || - jobPartStatus == DataTransferStatus.CompletedWithFailedTransfers) - && (jobStatus == DataTransferStatus.Queued || - jobStatus == DataTransferStatus.InProgress || - jobStatus == DataTransferStatus.PauseInProgress || - jobStatus == DataTransferStatus.CancellationInProgress)) + if ((jobPartStatus.State == DataTransferState.Paused || + jobPartStatus.State == DataTransferState.Completed) + && (jobState == DataTransferState.Queued || + jobState == DataTransferState.InProgress || + jobState == DataTransferState.Pausing || + jobState == DataTransferState.Stopping)) { Interlocked.Decrement(ref _pendingJobParts); @@ -379,84 +364,60 @@ public async Task JobPartEvent(TransferStatusEventArgs args) } } - public async Task OnJobStatusChangedAsync(DataTransferStatus status) + public async Task OnJobStateChangedAsync(DataTransferState state) { - bool statusChanged = false; - lock (_statusLock) - { - statusChanged = _dataTransfer._state.TrySetTransferStatus(status); - } - if (statusChanged) + if (_dataTransfer._state.TrySetTransferState(state)) { // If we are in a final state, dispose the JobPartEvent handlers - if (status == DataTransferStatus.Completed || - status == DataTransferStatus.CompletedWithSkippedTransfers || - status == DataTransferStatus.CompletedWithFailedTransfers || - status == DataTransferStatus.Paused) + if (state == DataTransferState.Completed || + state == DataTransferState.Paused) { DisposeHandlers(); } - if (TransferStatusEventHandler != null) - { - await TransferStatusEventHandler.RaiseAsync( - new TransferStatusEventArgs( - transferId: _dataTransfer.Id, - transferStatus: status, - isRunningSynchronously: false, - cancellationToken: _cancellationToken), - nameof(TransferJobInternal), - nameof(TransferStatusEventHandler), - ClientDiagnostics).ConfigureAwait(false); - } - await SetCheckpointerStatus(status).ConfigureAwait(false); + await OnJobPartStatusChangedAsync().ConfigureAwait(false); + await SetCheckpointerStatus().ConfigureAwait(false); } } - public async Task OnJobPartStatusChangedAsync(DataTransferStatus status) + public async Task OnJobPartStatusChangedAsync() { - //TODO: change to RaiseAsync after implementing ClientDiagnostics for TransferManager - await JobPartStatusEvents.RaiseAsync( - new TransferStatusEventArgs( - transferId: _dataTransfer.Id, - transferStatus: status, - isRunningSynchronously: false, - cancellationToken: _cancellationToken), - nameof(TransferJobInternal), - nameof(JobPartStatusEvents), - ClientDiagnostics) - .ConfigureAwait(false); + if (TransferStatusEventHandler != null) + { + await TransferStatusEventHandler.RaiseAsync( + new TransferStatusEventArgs( + transferId: _dataTransfer.Id, + transferStatus: _dataTransfer.TransferStatus.DeepCopy(), + isRunningSynchronously: false, + cancellationToken: _cancellationToken), + nameof(TransferJobInternal), + nameof(TransferStatusEventHandler), + ClientDiagnostics).ConfigureAwait(false); + } } - internal async virtual Task SetCheckpointerStatus(DataTransferStatus status) + internal async virtual Task SetCheckpointerStatus() { await _checkpointer.SetJobTransferStatusAsync( transferId: _dataTransfer.Id, - status: status).ConfigureAwait(false); + status: _dataTransfer.TransferStatus).ConfigureAwait(false); } internal async Task OnEnumerationComplete() { // If there were no job parts enumerated and we haven't already aborted/completed the job. if (_jobParts.Count == 0 && - _dataTransfer.TransferStatus != DataTransferStatus.Paused && - _dataTransfer.TransferStatus != DataTransferStatus.CompletedWithSkippedTransfers && - _dataTransfer.TransferStatus != DataTransferStatus.CompletedWithFailedTransfers && - _dataTransfer.TransferStatus != DataTransferStatus.Completed) + _dataTransfer.TransferStatus.State != DataTransferState.Paused && + _dataTransfer.TransferStatus.State != DataTransferState.Completed) { - if (_dataTransfer.TransferStatus == DataTransferStatus.PauseInProgress) + if (_dataTransfer.TransferStatus.State == DataTransferState.Pausing) { // If we paused before we were able to list, set the status properly. - await OnJobStatusChangedAsync(DataTransferStatus.Paused).ConfigureAwait(false); - } - else if (_dataTransfer.TransferStatus == DataTransferStatus.CancellationInProgress) - { - // If we aborted before we were able to list, set the status properly. - await OnJobStatusChangedAsync(DataTransferStatus.CompletedWithFailedTransfers).ConfigureAwait(false); + await OnJobStateChangedAsync(DataTransferState.Paused).ConfigureAwait(false); } else { - await OnJobStatusChangedAsync(DataTransferStatus.Completed).ConfigureAwait(false); + await OnJobStateChangedAsync(DataTransferState.Completed).ConfigureAwait(false); } } await CheckAndUpdateStatusAsync().ConfigureAwait(false); @@ -468,13 +429,13 @@ internal async Task CheckAndUpdateStatusAsync() // This is in the case that we weren't able to begin listing any job parts yet. if (_jobParts.Count == 0) { - if (_dataTransfer.TransferStatus == DataTransferStatus.PauseInProgress) + if (_dataTransfer.TransferStatus.State == DataTransferState.Pausing) { - await OnJobStatusChangedAsync(DataTransferStatus.Paused).ConfigureAwait(false); + await OnJobStateChangedAsync(DataTransferState.Paused).ConfigureAwait(false); } - else if (_dataTransfer.TransferStatus == DataTransferStatus.CancellationInProgress) + else if (_dataTransfer.TransferStatus.State == DataTransferState.Stopping) { - await OnJobStatusChangedAsync(DataTransferStatus.CompletedWithFailedTransfers).ConfigureAwait(false); + await OnJobStateChangedAsync(DataTransferState.Completed).ConfigureAwait(false); } return; } @@ -484,19 +445,11 @@ internal async Task CheckAndUpdateStatusAsync() { if (_jobPartPaused) { - await OnJobStatusChangedAsync(DataTransferStatus.Paused).ConfigureAwait(false); - } - else if (_jobPartFailed) - { - await OnJobStatusChangedAsync(DataTransferStatus.CompletedWithFailedTransfers).ConfigureAwait(false); - } - else if (_jobPartSkipped) - { - await OnJobStatusChangedAsync(DataTransferStatus.CompletedWithSkippedTransfers).ConfigureAwait(false); + await OnJobStateChangedAsync(DataTransferState.Paused).ConfigureAwait(false); } else { - await OnJobStatusChangedAsync(DataTransferStatus.Completed).ConfigureAwait(false); + await OnJobStateChangedAsync(DataTransferState.Completed).ConfigureAwait(false); } } } @@ -507,9 +460,7 @@ public void AppendJobPart(JobPartInternal jobPart) // Job parts can come from resuming a transfer and therefore may already be complete DataTransferStatus status = jobPart.JobPartStatus; - if (status != DataTransferStatus.CompletedWithSkippedTransfers && - status != DataTransferStatus.CompletedWithFailedTransfers && - status != DataTransferStatus.Completed) + if (status.State != DataTransferState.Completed) { Interlocked.Increment(ref _pendingJobParts); } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/UriToStreamJobPart.cs b/sdk/storage/Azure.Storage.DataMovement/src/UriToStreamJobPart.cs index 30258ce5f67e..7e80aed3afa0 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/UriToStreamJobPart.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/UriToStreamJobPart.cs @@ -58,7 +58,7 @@ private UriToStreamJobPart( StorageResourceItem sourceResource, StorageResourceItem destinationResource, bool isFinalPart, - DataTransferStatus jobPartStatus = DataTransferStatus.Queued, + DataTransferStatus jobPartStatus = default, long? length = default) : base(dataTransfer: job._dataTransfer, partNumber: partNumber, @@ -147,7 +147,7 @@ public override async Task ProcessPartToChunkAsync() { // we can default the length to 0 because we know the destination is local and // does not require a length to be created. - await OnTransferStatusChanged(DataTransferStatus.InProgress).ConfigureAwait(false); + await OnTransferStateChangedAsync(DataTransferState.InProgress).ConfigureAwait(false); try { @@ -247,7 +247,7 @@ await DownloadStreamingInternal(range: httpRange).ConfigureAwait(false)) } else { - await CheckAndUpdateCancellationStatusAsync().ConfigureAwait(false); + await CheckAndUpdateCancellationStateAsync().ConfigureAwait(false); } } catch (Exception ex) @@ -340,7 +340,7 @@ await _destinationResource.CompleteTransferAsync( await DisposeHandlers().ConfigureAwait(false); // Update the transfer status - await OnTransferStatusChanged(DataTransferStatus.Completed).ConfigureAwait(false); + await OnTransferStateChangedAsync(DataTransferState.Completed).ConfigureAwait(false); } catch (Exception ex) { @@ -512,7 +512,7 @@ await QueueChunkToChannelAsync( } else { - await CheckAndUpdateCancellationStatusAsync().ConfigureAwait(false); + await CheckAndUpdateCancellationStateAsync().ConfigureAwait(false); } } } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/UriToStreamTransferJob.cs b/sdk/storage/Azure.Storage.DataMovement/src/UriToStreamTransferJob.cs index 7ed248d5c5b7..67bcdf915c99 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/UriToStreamTransferJob.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/UriToStreamTransferJob.cs @@ -67,7 +67,7 @@ internal UriToStreamTransferJob( /// An IEnumerable that contains the job parts public override async IAsyncEnumerable ProcessJobToJobPartAsync() { - await OnJobStatusChangedAsync(DataTransferStatus.InProgress).ConfigureAwait(false); + await OnJobStateChangedAsync(DataTransferState.InProgress).ConfigureAwait(false); int partNumber = 0; if (_jobParts.Count == 0) @@ -106,9 +106,9 @@ public override async IAsyncEnumerable ProcessJobToJobPartAsync bool isFinalPartFound = false; foreach (JobPartInternal part in _jobParts) { - if (part.JobPartStatus != DataTransferStatus.Completed) + if (!part.JobPartStatus.HasCompletedSuccessfully) { - part.JobPartStatus = DataTransferStatus.Queued; + part.JobPartStatus.TrySetTransferStateChange(DataTransferState.Queued); yield return part; if (part.IsFinalPart) diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/DataTransferTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/DataTransferTests.cs index 879586d0c8b6..215a5d961714 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/DataTransferTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/DataTransferTests.cs @@ -12,6 +12,9 @@ namespace Azure.Storage.DataMovement.Tests public class DataTransferTests { private static string GetNewTransferId() => Guid.NewGuid().ToString(); + private static DataTransferStatus QueuedStatus => new DataTransferStatus(DataTransferState.Queued, false, false); + private static DataTransferStatus InProgressStatus => new DataTransferStatus(DataTransferState.InProgress, false, false); + private static DataTransferStatus SuccessfulCompletedStatus => new DataTransferStatus(DataTransferState.Completed, false, false); [Test] public void Ctor_Default() @@ -30,13 +33,14 @@ public void Ctor_Default() } [Test] - [TestCase(DataTransferStatus.None)] - [TestCase(DataTransferStatus.Queued)] - [TestCase(DataTransferStatus.InProgress)] - [TestCase(DataTransferStatus.PauseInProgress)] - [TestCase(DataTransferStatus.CancellationInProgress)] - [TestCase(DataTransferStatus.Paused)] - public void HasCompleted_False(DataTransferStatus status) + [TestCase(DataTransferState.None, false)] + [TestCase(DataTransferState.Queued, false)] + [TestCase(DataTransferState.InProgress, false)] + [TestCase(DataTransferState.Pausing, false)] + [TestCase(DataTransferState.Paused, false)] + [TestCase(DataTransferState.Stopping, false)] + [TestCase(DataTransferState.Stopping, true)] + public void HasCompleted_False(DataTransferState status, bool hasFailedItems) { // Arrange string transferId = GetNewTransferId(); @@ -46,7 +50,7 @@ public void HasCompleted_False(DataTransferStatus status) DataTransfer transfer = new DataTransfer( id: transferId, transferManager: transferManager, - status: status); + status: new DataTransferStatus(status, hasFailedItems, false)); // Assert Assert.AreEqual(transferId, transfer.Id); @@ -55,10 +59,14 @@ public void HasCompleted_False(DataTransferStatus status) } [Test] - [TestCase(DataTransferStatus.Completed)] - [TestCase(DataTransferStatus.CompletedWithSkippedTransfers)] - [TestCase(DataTransferStatus.CompletedWithFailedTransfers)] - public void HasCompleted_True(DataTransferStatus status) + [TestCase(DataTransferState.Completed, false, false)] + [TestCase(DataTransferState.Completed, false, true)] + [TestCase(DataTransferState.Completed, true, false)] + [TestCase(DataTransferState.Completed, true, true)] + public void HasCompleted_True( + DataTransferState state, + bool hasFailedItems, + bool hasSkippedItems) { // Arrange string transferId = GetNewTransferId(); @@ -68,7 +76,7 @@ public void HasCompleted_True(DataTransferStatus status) DataTransfer transfer = new DataTransfer( id: transferId, transferManager: transferManager, - status: status); + status: new DataTransferStatus(state, hasFailedItems, hasSkippedItems)); // Assert Assert.AreEqual(transferId, transfer.Id); @@ -86,7 +94,7 @@ public void EnsureCompleted() DataTransfer transfer = new DataTransfer( id: transferId, transferManager: transferManager, - status: DataTransferStatus.Completed); + status: SuccessfulCompletedStatus); // Act transfer.WaitForCompletion(); @@ -106,7 +114,7 @@ public void EnsureCompleted_CancellationToken() DataTransfer transfer = new DataTransfer( id: transferId, transferManager: new(), - status: DataTransferStatus.Queued); + status: QueuedStatus); CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(1)); // Act @@ -125,7 +133,7 @@ public async Task AwaitCompletion() DataTransfer transfer = new DataTransfer( id: transferId, transferManager: transferManager, - status: DataTransferStatus.Completed); + status: SuccessfulCompletedStatus); // Act await transfer.WaitForCompletionAsync(); @@ -145,7 +153,7 @@ public void AwaitCompletion_CancellationToken() DataTransfer transfer = new DataTransfer( id: transferId, transferManager: new(), - status: DataTransferStatus.Queued); + status: QueuedStatus); CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(1)); Assert.CatchAsync( @@ -162,43 +170,49 @@ public async Task TryPauseAsync() DataTransfer transfer = new DataTransfer( id: transferId, transferManager: new(), - status: DataTransferStatus.InProgress); + status: InProgressStatus); // Act Task pauseTask = transfer.PauseAsync(); - Assert.AreEqual(DataTransferStatus.PauseInProgress, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Pausing, transfer.TransferStatus.State); // Assert - if (!transfer._state.TrySetTransferStatus(DataTransferStatus.Paused)) + if (!transfer._state.TrySetTransferState(DataTransferState.Paused)) { Assert.Fail("Unable to set the transfer status internally to the DataTransfer."); } await pauseTask; - Assert.AreEqual(DataTransferStatus.Paused, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Paused, transfer.TransferStatus.State); Assert.IsFalse(transfer.HasCompleted); } [Test] - [TestCase(DataTransferStatus.Paused)] - [TestCase(DataTransferStatus.Completed)] - [TestCase(DataTransferStatus.CompletedWithSkippedTransfers)] - [TestCase(DataTransferStatus.CompletedWithFailedTransfers)] - public async Task TryPauseAsync_AlreadyPaused(DataTransferStatus status) + [TestCase(DataTransferState.Paused, false, false)] + [TestCase(DataTransferState.Completed, false, false)] + [TestCase(DataTransferState.Completed, false, true)] + [TestCase(DataTransferState.Completed, true, false)] + [TestCase(DataTransferState.Completed, true, true)] + public async Task TryPauseAsync_AlreadyPaused( + DataTransferState state, + bool hasFailedItems, + bool hasSkippedItems) { // Arrange string transferId = GetNewTransferId(); + DataTransferStatus originalStatus = new DataTransferStatus(state, hasFailedItems, hasSkippedItems); + DataTransfer transfer = new DataTransfer( id: transferId, transferManager: new(), - status: status); + status: originalStatus); - Assert.AreEqual(status, transfer.TransferStatus); + Assert.AreEqual(originalStatus, transfer.TransferStatus); await transfer.PauseAsync(); - Assert.AreEqual(status, transfer.TransferStatus); + Assert.AreEqual(originalStatus, transfer.TransferStatus); } [Test] @@ -210,7 +224,7 @@ public async Task TryPauseAsync_CancellationToken() DataTransfer transfer = new DataTransfer( id: transferId, transferManager: new(), - status: DataTransferStatus.InProgress); + status: InProgressStatus); CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(1)); try @@ -221,7 +235,7 @@ public async Task TryPauseAsync_CancellationToken() { Assert.AreEqual(exception.Message, "The operation was canceled."); } - Assert.AreEqual(DataTransferStatus.PauseInProgress, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Pausing, transfer.TransferStatus.State); Assert.IsFalse(transfer.HasCompleted); } } diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/GetTransfersTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/GetTransfersTests.cs index 06939c2fd71f..8bae1edbd202 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/GetTransfersTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/GetTransfersTests.cs @@ -14,6 +14,14 @@ namespace Azure.Storage.DataMovement.Tests /// public class GetTransfersTests { + private static DataTransferStatus QueuedStatus => new DataTransferStatus(DataTransferState.Queued, false, false); + private static DataTransferStatus InProgressStatus => new DataTransferStatus(DataTransferState.InProgress, false, false); + private static DataTransferStatus PausedStatus => new DataTransferStatus(DataTransferState.Paused, false, false); + private static DataTransferStatus SuccessfulCompletedStatus => new DataTransferStatus(DataTransferState.Completed, false, false); + private static DataTransferStatus FailedCompletedStatus => new DataTransferStatus(DataTransferState.Completed, true, false); + private static DataTransferStatus SkippedCompletedStatus => new DataTransferStatus(DataTransferState.Completed, true, false); + private static DataTransferStatus FailedSkippedCompletedStatus => new DataTransferStatus(DataTransferState.Completed, true, false); + private TransferManagerOptions GetDefaultManagerOptions(string checkpointerPath) => new TransferManagerOptions() { @@ -38,7 +46,7 @@ private void AssertListTransfersEquals(IList expected, IList storedTransfers = new List { - GetNewDataTransfer(DataTransferStatus.Queued), - GetNewDataTransfer(DataTransferStatus.Queued), - GetNewDataTransfer(DataTransferStatus.Queued), - GetNewDataTransfer(DataTransferStatus.Queued), - GetNewDataTransfer(DataTransferStatus.InProgress), - GetNewDataTransfer(DataTransferStatus.InProgress), - GetNewDataTransfer(DataTransferStatus.Paused), - GetNewDataTransfer(DataTransferStatus.Paused), - GetNewDataTransfer(DataTransferStatus.CompletedWithFailedTransfers), - GetNewDataTransfer(DataTransferStatus.CompletedWithFailedTransfers), - GetNewDataTransfer(DataTransferStatus.CompletedWithFailedTransfers), - GetNewDataTransfer(DataTransferStatus.Completed), - GetNewDataTransfer(DataTransferStatus.Completed), + GetNewDataTransfer(QueuedStatus), + GetNewDataTransfer(QueuedStatus), + GetNewDataTransfer(QueuedStatus), + GetNewDataTransfer(QueuedStatus), + GetNewDataTransfer(InProgressStatus), + GetNewDataTransfer(InProgressStatus), + GetNewDataTransfer(PausedStatus), + GetNewDataTransfer(PausedStatus), + GetNewDataTransfer(FailedCompletedStatus), + GetNewDataTransfer(FailedCompletedStatus), + GetNewDataTransfer(FailedCompletedStatus), + GetNewDataTransfer(SuccessfulCompletedStatus), + GetNewDataTransfer(SuccessfulCompletedStatus), }; TransferManagerFactory factory = new TransferManagerFactory(GetDefaultManagerOptions(testDirectory.DirectoryPath)); TransferManager manager = factory.BuildTransferManager(storedTransfers); // Act - DataTransferStatus[] statuses = new DataTransferStatus[] { status }; - IList result = await manager.GetTransfersAsync(statuses).ToListAsync(); + DataTransferStatus status = new DataTransferStatus(state, hasFailedItems, hasSkippedItems); + IList result = await manager.GetTransfersAsync(status).ToListAsync(); // Assert AssertListTransfersEquals(storedTransfers.Where( d => d.TransferStatus == status).ToList(), result); @@ -127,20 +138,20 @@ public async Task GetTransfers_FilterMultipleStatuses() // Arrange - Set up transfer manager with multiple transfers List storedTransfers = new List { - GetNewDataTransfer(DataTransferStatus.Queued), - GetNewDataTransfer(DataTransferStatus.Queued), - GetNewDataTransfer(DataTransferStatus.Queued), - GetNewDataTransfer(DataTransferStatus.Queued), - GetNewDataTransfer(DataTransferStatus.InProgress), - GetNewDataTransfer(DataTransferStatus.InProgress), - GetNewDataTransfer(DataTransferStatus.Paused), - GetNewDataTransfer(DataTransferStatus.Paused), - GetNewDataTransfer(DataTransferStatus.CompletedWithFailedTransfers), - GetNewDataTransfer(DataTransferStatus.CompletedWithFailedTransfers), - GetNewDataTransfer(DataTransferStatus.CompletedWithFailedTransfers), - GetNewDataTransfer(DataTransferStatus.Completed), - GetNewDataTransfer(DataTransferStatus.Completed), - GetNewDataTransfer(DataTransferStatus.CompletedWithSkippedTransfers) + GetNewDataTransfer(QueuedStatus), + GetNewDataTransfer(QueuedStatus), + GetNewDataTransfer(QueuedStatus), + GetNewDataTransfer(QueuedStatus), + GetNewDataTransfer(InProgressStatus), + GetNewDataTransfer(InProgressStatus), + GetNewDataTransfer(PausedStatus), + GetNewDataTransfer(PausedStatus), + GetNewDataTransfer(FailedCompletedStatus), + GetNewDataTransfer(FailedCompletedStatus), + GetNewDataTransfer(FailedCompletedStatus), + GetNewDataTransfer(SuccessfulCompletedStatus), + GetNewDataTransfer(SuccessfulCompletedStatus), + GetNewDataTransfer(SkippedCompletedStatus) }; TransferManagerFactory factory = new TransferManagerFactory(GetDefaultManagerOptions(testDirectory.DirectoryPath)); @@ -148,9 +159,9 @@ public async Task GetTransfers_FilterMultipleStatuses() // Act DataTransferStatus[] statuses = new DataTransferStatus[] { - DataTransferStatus.Completed, - DataTransferStatus.CompletedWithFailedTransfers, - DataTransferStatus.CompletedWithSkippedTransfers }; + SuccessfulCompletedStatus, + FailedCompletedStatus, + SkippedCompletedStatus }; IList result = await manager.GetTransfersAsync(statuses).ToListAsync(); // Assert @@ -164,26 +175,26 @@ public async Task GetTransfers_Filtered_Empty() // Arrange - Set up transfer manager with multiple transfers List storedTransfers = new List { - GetNewDataTransfer(DataTransferStatus.Queued), - GetNewDataTransfer(DataTransferStatus.Queued), - GetNewDataTransfer(DataTransferStatus.Queued), - GetNewDataTransfer(DataTransferStatus.Queued), - GetNewDataTransfer(DataTransferStatus.InProgress), - GetNewDataTransfer(DataTransferStatus.InProgress), - GetNewDataTransfer(DataTransferStatus.Paused), - GetNewDataTransfer(DataTransferStatus.Paused), - GetNewDataTransfer(DataTransferStatus.CompletedWithFailedTransfers), - GetNewDataTransfer(DataTransferStatus.CompletedWithFailedTransfers), - GetNewDataTransfer(DataTransferStatus.CompletedWithFailedTransfers), - GetNewDataTransfer(DataTransferStatus.Completed), - GetNewDataTransfer(DataTransferStatus.Completed), + GetNewDataTransfer(QueuedStatus), + GetNewDataTransfer(QueuedStatus), + GetNewDataTransfer(QueuedStatus), + GetNewDataTransfer(QueuedStatus), + GetNewDataTransfer(InProgressStatus), + GetNewDataTransfer(InProgressStatus), + GetNewDataTransfer(PausedStatus), + GetNewDataTransfer(PausedStatus), + GetNewDataTransfer(FailedCompletedStatus), + GetNewDataTransfer(FailedCompletedStatus), + GetNewDataTransfer(FailedCompletedStatus), + GetNewDataTransfer(SuccessfulCompletedStatus), + GetNewDataTransfer(SuccessfulCompletedStatus), }; TransferManagerFactory factory = new TransferManagerFactory(GetDefaultManagerOptions(testDirectory.DirectoryPath)); TransferManager manager = factory.BuildTransferManager(storedTransfers); // Act - With a transfer status not in the above stored transfers - DataTransferStatus[] statuses = new DataTransferStatus[] { DataTransferStatus.CancellationInProgress }; + DataTransferStatus[] statuses = new DataTransferStatus[] { new DataTransferStatus(DataTransferState.Stopping, true, false) }; IList result = await manager.GetTransfersAsync(statuses).ToListAsync(); // Assert @@ -274,14 +285,14 @@ public async Task GetResumableTransfers_IgnoresCompleted() test.DirectoryPath, transferId1, 3 /* jobPartCount */, - DataTransferStatus.Completed); + SuccessfulCompletedStatus); string transferId2 = Guid.NewGuid().ToString(); factory.CreateStubJobPartPlanFilesAsync( test.DirectoryPath, transferId2, 3 /* jobPartCount */, - DataTransferStatus.Queued); + QueuedStatus); // Build TransferManager with the stored transfers TransferManagerOptions options = new TransferManagerOptions() @@ -326,7 +337,7 @@ private void AddTransferFromDataTransferProperties( checkpointerPath, properties.TransferId, numParts, /* jobPartCount */ - DataTransferStatus.InProgress, + InProgressStatus, sourcePaths, destinationPaths, sourceResourceId: properties.SourceTypeId, @@ -338,7 +349,7 @@ private void AddTransferFromDataTransferProperties( checkpointerPath, properties.TransferId, 1, /* jobPartCount */ - DataTransferStatus.InProgress, + InProgressStatus, new List { properties.SourcePath }, new List { properties.DestinationPath }, sourceResourceId: properties.SourceTypeId, diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/JobPartPlanHeaderTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/JobPartPlanHeaderTests.cs index 1fbf6fd1fd2c..cd555c094247 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/JobPartPlanHeaderTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/JobPartPlanHeaderTests.cs @@ -395,13 +395,29 @@ public async Task Serialize() await stream.ReadAsync(rehydratePriorityTypeBuffer, 0, oneByte); Assert.AreEqual((byte)DefaultRehydratePriorityType, rehydratePriorityTypeBuffer[0]); - byte[] atomicJobStatusBuffer = new byte[oneByte]; - await stream.ReadAsync(atomicJobStatusBuffer, 0, oneByte); - Assert.AreEqual((byte)DefaultJobStatus, atomicJobStatusBuffer[0]); + byte[] atomicJobStateBuffer = new byte[oneByte]; + await stream.ReadAsync(atomicJobStateBuffer, 0, oneByte); + Assert.AreEqual((byte)DefaultJobStatus.State, atomicJobStateBuffer[0]); - byte[] atomicPartStatusBuffer = new byte[oneByte]; - await stream.ReadAsync(atomicPartStatusBuffer, 0, oneByte); - Assert.AreEqual((byte)DefaultPartStatus, atomicPartStatusBuffer[0]); + byte[] atomicJobHasFailedItemsBuffer = new byte[oneByte]; + await stream.ReadAsync(atomicJobHasFailedItemsBuffer, 0, oneByte); + Assert.AreEqual(Convert.ToByte(DefaultJobStatus.HasFailedItems), atomicJobHasFailedItemsBuffer[0]); + + byte[] atomicJobHasSkippedItemsBuffer = new byte[oneByte]; + await stream.ReadAsync(atomicJobHasSkippedItemsBuffer, 0, oneByte); + Assert.AreEqual(Convert.ToByte(DefaultJobStatus.HasSkippedItems), atomicJobHasSkippedItemsBuffer[0]); + + byte[] atomicPartStateBuffer = new byte[oneByte]; + await stream.ReadAsync(atomicPartStateBuffer, 0, oneByte); + Assert.AreEqual((byte)DefaultPartStatus.State, atomicPartStateBuffer[0]); + + byte[] atomiPartHasFailedItemsBuffer = new byte[oneByte]; + await stream.ReadAsync(atomiPartHasFailedItemsBuffer, 0, oneByte); + Assert.AreEqual(Convert.ToByte(DefaultJobStatus.HasFailedItems), atomiPartHasFailedItemsBuffer[0]); + + byte[] atomicPartHasSkippedItemsBuffer = new byte[oneByte]; + await stream.ReadAsync(atomicPartHasSkippedItemsBuffer, 0, oneByte); + Assert.AreEqual(Convert.ToByte(DefaultJobStatus.HasSkippedItems), atomicPartHasSkippedItemsBuffer[0]); } } @@ -418,7 +434,7 @@ public void Serialize_Error() } [Test] - public void Deserialize() + public void Deserialize() { // Arrange IDictionary metadata = DataProvider.BuildMetadata(); @@ -459,7 +475,21 @@ public void Deserialize_File_Version_b2() using (FileStream stream = File.OpenRead(samplePath)) { // Act / Assert - DeserializeAndVerify(stream, DataMovementConstants.JobPartPlanFile.SchemaVersion_b2, DataProvider.BuildMetadata(), DataProvider.BuildTags()); + Assert.Catch( + () => JobPartPlanHeader.Deserialize(stream), + $"The checkpoint file schema version {DataMovementConstants.JobPartPlanFile.SchemaVersion_b2} is not supported by this version of the SDK."); + } + } + + [Test] + public void Deserialize_File_Version_b3() + { + // Arrange + string samplePath = Path.Combine("Resources", "SampleJobPartPlanFile.steVb3"); + using (FileStream stream = File.OpenRead(samplePath)) + { + // Act / Assert + DeserializeAndVerify(stream, DataMovementConstants.JobPartPlanFile.SchemaVersion_b3, DataProvider.BuildMetadata(), DataProvider.BuildTags()); } } @@ -540,8 +570,12 @@ private void DeserializeAndVerify( Assert.AreEqual(deserializedHeader.DeleteSnapshotsOption, DefaultDeleteSnapshotsOption); Assert.AreEqual(deserializedHeader.PermanentDeleteOption, DefaultPermanentDeleteOption); Assert.AreEqual(deserializedHeader.RehydratePriorityType, DefaultRehydratePriorityType); - Assert.AreEqual(deserializedHeader.AtomicJobStatus, DefaultJobStatus); - Assert.AreEqual(deserializedHeader.AtomicPartStatus, DefaultPartStatus); + Assert.AreEqual(DefaultJobStatus.State, deserializedHeader.AtomicJobStatus.State); + Assert.AreEqual(DefaultJobStatus.HasFailedItems, deserializedHeader.AtomicJobStatus.HasFailedItems); + Assert.AreEqual(DefaultJobStatus.HasSkippedItems, deserializedHeader.AtomicJobStatus.HasSkippedItems); + Assert.AreEqual(DefaultPartStatus.State, deserializedHeader.AtomicPartStatus.State); + Assert.AreEqual(DefaultPartStatus.HasFailedItems, deserializedHeader.AtomicPartStatus.HasFailedItems); + Assert.AreEqual(DefaultPartStatus.HasSkippedItems, deserializedHeader.AtomicPartStatus.HasSkippedItems); } } } diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/LocalTransferCheckpointerFactory.cs b/sdk/storage/Azure.Storage.DataMovement/tests/LocalTransferCheckpointerFactory.cs index e491f92ecf08..2f6583895c7f 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/LocalTransferCheckpointerFactory.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/LocalTransferCheckpointerFactory.cs @@ -43,8 +43,8 @@ internal static readonly DateTimeOffset _testStartTime internal const JobPartDeleteSnapshotsOption _testDeleteSnapshotsOption = JobPartDeleteSnapshotsOption.None; internal const JobPartPermanentDeleteOption _testPermanentDeleteOption = JobPartPermanentDeleteOption.None; internal const JobPartPlanRehydratePriorityType _testRehydratePriorityType = JobPartPlanRehydratePriorityType.None; - internal const DataTransferStatus _testJobStatus = DataTransferStatus.Queued; - internal const DataTransferStatus _testPartStatus = DataTransferStatus.Queued; + internal static readonly DataTransferStatus _testJobStatus = new DataTransferStatusInternal(DataTransferState.Queued, false, false); + internal static readonly DataTransferStatus _testPartStatus = new DataTransferStatusInternal(DataTransferState.Queued, false, false); private string _checkpointerPath; @@ -89,12 +89,13 @@ internal void CreateStubJobPartPlanFilesAsync( string checkpointerPath, string transferId, int jobPartCount, - DataTransferStatus status = DataTransferStatus.Queued, + DataTransferStatus status = default, List sourcePaths = default, List destinationPaths = default, string sourceResourceId = "LocalFile", string destinationResourceId = "LocalFile") { + status ??= _testPartStatus; // Populate sourcePaths if not provided if (sourcePaths == default) { @@ -185,9 +186,11 @@ internal JobPartPlanHeader CreateDefaultJobPartHeader( JobPartDeleteSnapshotsOption deleteSnapshotsOption = _testDeleteSnapshotsOption, JobPartPermanentDeleteOption permanentDeleteOption = _testPermanentDeleteOption, JobPartPlanRehydratePriorityType rehydratePriorityType = _testRehydratePriorityType, - DataTransferStatus atomicJobStatus = _testJobStatus, - DataTransferStatus atomicPartStatus = _testPartStatus) + DataTransferStatus atomicJobStatus = default, + DataTransferStatus atomicPartStatus = default) { + atomicJobStatus ??= _testJobStatus; + atomicPartStatus ??= _testPartStatus; if (startTime == default) { startTime = _testStartTime; diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/LocalTransferCheckpointerTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/LocalTransferCheckpointerTests.cs index 2c5539e160d3..3b38efe6f964 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/LocalTransferCheckpointerTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/LocalTransferCheckpointerTests.cs @@ -14,6 +14,8 @@ namespace Azure.Storage.DataMovement.Tests { public class LocalTransferCheckpointerTests : DataMovementTestBase { + private static DataTransferStatus SuccessfulCompletedStatus => new DataTransferStatus(DataTransferState.Completed, false, false); + public LocalTransferCheckpointerTests(bool async) : base(async, null) { @@ -906,7 +908,7 @@ public async Task SetJobTransferStatusAsync() string transferId = GetNewTransferId(); int partNumber = 0; int chunksTotal = 1; - DataTransferStatus newStatus = DataTransferStatus.Completed; + DataTransferStatus newStatus = SuccessfulCompletedStatus; JobPartPlanHeader header = CheckpointerTesting.CreateDefaultJobPartHeader( transferId: transferId, partNumber: partNumber); @@ -948,7 +950,7 @@ public async Task SetJobTransferStatusAsync_MultipleParts() // Arrange string transferId = GetNewTransferId(); int chunksTotal = 1; - DataTransferStatus newStatus = DataTransferStatus.Completed; + DataTransferStatus newStatus = SuccessfulCompletedStatus; JobPartPlanHeader header1 = CheckpointerTesting.CreateDefaultJobPartHeader( transferId: transferId, partNumber: 0); @@ -1057,7 +1059,7 @@ public void SetJobTransferStatusAsync_Error() // Arrange string transferId = GetNewTransferId(); int partNumber = 0; - DataTransferStatus newStatus = DataTransferStatus.Completed; + DataTransferStatus newStatus = SuccessfulCompletedStatus; JobPartPlanHeader header = CheckpointerTesting.CreateDefaultJobPartHeader( transferId: transferId, partNumber: partNumber); @@ -1079,7 +1081,7 @@ public async Task SetJobPartTransferStatusAsync() int partNumber = 0; int chunksTotal = 1; // originally the default is set to Queued - DataTransferStatus newStatus = DataTransferStatus.Completed; + DataTransferStatus newStatus = SuccessfulCompletedStatus; JobPartPlanHeader header = CheckpointerTesting.CreateDefaultJobPartHeader( transferId: transferId, partNumber: partNumber); @@ -1122,7 +1124,7 @@ public void SetJobPartTransferStatusAsync_Error() string transferId = GetNewTransferId(); int partNumber = 0; // originally the default is set to Queued - DataTransferStatus newStatus = DataTransferStatus.Completed; + DataTransferStatus newStatus = SuccessfulCompletedStatus; JobPartPlanHeader header = CheckpointerTesting.CreateDefaultJobPartHeader( transferId: transferId, partNumber: partNumber); diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/PauseResumeTransferTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/PauseResumeTransferTests.cs index bfd2b3350225..c474ab1a1a83 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/PauseResumeTransferTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/PauseResumeTransferTests.cs @@ -256,7 +256,7 @@ public async Task TryPauseTransferAsync_Id(TransferDirection transferType) // Assert await testEventsRaised.AssertPausedCheck(); - Assert.AreEqual(DataTransferStatus.Paused, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Paused, transfer.TransferStatus.State); // Check if Job Plan File exists in checkpointer path. JobPartPlanFileName fileName = new JobPartPlanFileName( @@ -304,7 +304,7 @@ public async Task TryPauseTransferAsync_DataTransfer(TransferDirection transferT // Assert await testEventsRaised.AssertPausedCheck(); - Assert.AreEqual(DataTransferStatus.Paused, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Paused, transfer.TransferStatus.State); // Check if Job Plan File exists in checkpointer path. JobPartPlanFileName fileName = new JobPartPlanFileName( @@ -368,12 +368,12 @@ public async Task TryPauseTransferAsync_AlreadyPaused(TransferDirection transfer // Assert await testEventsRaised.AssertPausedCheck(); - Assert.AreEqual(DataTransferStatus.Paused, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Paused, transfer.TransferStatus.State); CancellationTokenSource cancellationTokenSource2 = new CancellationTokenSource(TimeSpan.FromSeconds(10)); await transferManager.PauseTransferIfRunningAsync(transfer.Id, cancellationTokenSource2.Token); - Assert.AreEqual(DataTransferStatus.Paused, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Paused, transfer.TransferStatus.State); // Check if Job Plan File exists in checkpointer path. JobPartPlanFileName fileName = new JobPartPlanFileName( @@ -429,7 +429,7 @@ public async Task PauseThenResumeTransferAsync(TransferDirection transferType) await transferManager.PauseTransferIfRunningAsync(transfer.Id, cancellationTokenSource.Token); // Assert - Confirm we've paused - Assert.AreEqual(DataTransferStatus.Paused, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Paused, transfer.TransferStatus.State); await testEventsRaised.AssertPausedCheck(); // Act - Resume Job @@ -444,7 +444,7 @@ public async Task PauseThenResumeTransferAsync(TransferDirection transferType) // Assert await testEventRaised2.AssertSingleCompletedCheck(); - Assert.AreEqual(DataTransferStatus.Completed, resumeTransfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, resumeTransfer.TransferStatus.State); Assert.IsTrue(resumeTransfer.HasCompleted); // Verify transfer @@ -500,7 +500,7 @@ public async Task ResumeTransferAsync(TransferDirection transferType) await transferManager.PauseTransferIfRunningAsync(transfer.Id, cancellationTokenSource.Token); // Assert - Confirm we've paused - Assert.AreEqual(DataTransferStatus.Paused, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Paused, transfer.TransferStatus.State); await testEventsRaised.AssertPausedCheck(); // Act - Resume Job @@ -515,7 +515,7 @@ public async Task ResumeTransferAsync(TransferDirection transferType) // Assert await testEventRaised2.AssertSingleCompletedCheck(); - Assert.AreEqual(DataTransferStatus.Completed, resumeTransfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, resumeTransfer.TransferStatus.State); Assert.IsTrue(resumeTransfer.HasCompleted); // Verify transfer @@ -705,7 +705,7 @@ public async Task TryPauseTransferAsync_Id_Directory(TransferDirection transferT // Assert await testEventsRaised.AssertPausedCheck(); - Assert.AreEqual(DataTransferStatus.Paused, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Paused, transfer.TransferStatus.State); } [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] @@ -750,7 +750,7 @@ public async Task TryPauseTransferAsync_DataTransfer_Directory(TransferDirection // Assert await testEventsRaised.AssertPausedCheck(); - Assert.AreEqual(DataTransferStatus.Paused, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Paused, transfer.TransferStatus.State); } [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] @@ -795,13 +795,13 @@ public async Task TryPauseTransferAsync_AlreadyPaused_Directory(TransferDirectio // Assert await testEventsRaised.AssertPausedCheck(); - Assert.AreEqual(DataTransferStatus.Paused, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Paused, transfer.TransferStatus.State); CancellationTokenSource cancellationTokenSource2 = new CancellationTokenSource(TimeSpan.FromSeconds(10)); await transferManager.PauseTransferIfRunningAsync(transfer.Id, cancellationTokenSource2.Token); await testEventsRaised.AssertPausedCheck(); - Assert.AreEqual(DataTransferStatus.Paused, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Paused, transfer.TransferStatus.State); } [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] @@ -853,7 +853,7 @@ public async Task PauseThenResumeTransferAsync_Directory(TransferDirection trans await transferManager.PauseTransferIfRunningAsync(transfer.Id, cancellationTokenSource.Token); // Assert - Confirm we've paused - Assert.AreEqual(DataTransferStatus.Paused, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Paused, transfer.TransferStatus.State); await testEventsRaised.AssertPausedCheck(); // Act - Resume Job @@ -868,7 +868,7 @@ public async Task PauseThenResumeTransferAsync_Directory(TransferDirection trans // Assert await testEventRaised2.AssertContainerCompletedCheck(partCount); - Assert.AreEqual(DataTransferStatus.Completed, resumeTransfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, resumeTransfer.TransferStatus.State); Assert.IsTrue(resumeTransfer.HasCompleted); // Verify transfer @@ -929,7 +929,7 @@ public async Task ResumeTransferAsync_Directory(TransferDirection transferType) await transferManager.PauseTransferIfRunningAsync(transfer.Id, cancellationTokenSource.Token); // Assert - Confirm we've paused - Assert.AreEqual(DataTransferStatus.Paused, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Paused, transfer.TransferStatus.State); await testEventsRaised.AssertPausedCheck(); // Act - Resume Job @@ -944,7 +944,7 @@ public async Task ResumeTransferAsync_Directory(TransferDirection transferType) // Assert await testEventsRaised2.AssertContainerCompletedCheck(partCount); - Assert.AreEqual(DataTransferStatus.Completed, resumeTransfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, resumeTransfer.TransferStatus.State); Assert.IsTrue(resumeTransfer.HasCompleted); // Verify transfer @@ -962,9 +962,9 @@ public async Task PauseAllTriggersCorrectPauses() List> pausable = new(); List> unpausable = new(); TransferManager manager = new(); - foreach (DataTransferStatus state in Enum.GetValues(typeof(DataTransferStatus)).Cast()) + foreach (DataTransferState state in Enum.GetValues(typeof(DataTransferState)).Cast()) { - bool canPause = state == DataTransferStatus.InProgress; + bool canPause = state == DataTransferState.InProgress; Mock transfer = new(MockBehavior.Loose) { CallBase = true, diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/ProgressHandlerTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/ProgressHandlerTests.cs index 2e5b8af06442..8ca12a5b75d4 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/ProgressHandlerTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/ProgressHandlerTests.cs @@ -339,7 +339,7 @@ public async Task ProgressHandler_PauseResume(int delayInMs) // Pause transfer CancellationTokenSource tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); await transferManager.PauseTransferIfRunningAsync(transfer.Id, tokenSource.Token); - Assert.AreEqual(DataTransferStatus.Paused, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Paused, transfer.TransferStatus); // Record the current number of progress updates to use during assertions int pause = progressHandler.Updates.Count; @@ -353,7 +353,7 @@ public async Task ProgressHandler_PauseResume(int delayInMs) await resumeTransfer.WaitForCompletionAsync(tokenSource.Token); // Assert - Assert.AreEqual(DataTransferStatus.Completed, resumeTransfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, resumeTransfer.TransferStatus); ProgressHandlerAsserts.AssertFileProgress(progressHandler.Updates, 5, pauseIndexes: pause); ProgressHandlerAsserts.AssertBytesTransferred(progressHandler.Updates, _expectedBytesTransferred); } diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Resources/SampleJobPartPlanFile.steVb3 b/sdk/storage/Azure.Storage.DataMovement/tests/Resources/SampleJobPartPlanFile.steVb3 new file mode 100644 index 000000000000..01f7674be788 Binary files /dev/null and b/sdk/storage/Azure.Storage.DataMovement/tests/Resources/SampleJobPartPlanFile.steVb3 differ diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/CheckpointerTesting.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/CheckpointerTesting.cs index 319db3cabf38..57bfa4c9048c 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/CheckpointerTesting.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/CheckpointerTesting.cs @@ -52,8 +52,8 @@ internal static readonly DateTimeOffset DefaultStartTime internal const JobPartDeleteSnapshotsOption DefaultDeleteSnapshotsOption = JobPartDeleteSnapshotsOption.None; internal const JobPartPermanentDeleteOption DefaultPermanentDeleteOption = JobPartPermanentDeleteOption.None; internal const JobPartPlanRehydratePriorityType DefaultRehydratePriorityType = JobPartPlanRehydratePriorityType.None; - internal const DataTransferStatus DefaultJobStatus = DataTransferStatus.Queued; - internal const DataTransferStatus DefaultPartStatus = DataTransferStatus.Queued; + internal static readonly DataTransferStatus DefaultJobStatus = new DataTransferStatusInternal(DataTransferState.Queued, false, false); + internal static readonly DataTransferStatus DefaultPartStatus = new DataTransferStatusInternal(DataTransferState.Queued, false, false); internal static readonly DateTimeOffset DefaultCreateTime = new DateTimeOffset(2023, 08, 28, 17, 26, 0, default); internal const JobPlanStatus DefaultJobPlanStatus = JobPlanStatus.Queued; @@ -103,8 +103,8 @@ internal static JobPartPlanHeader CreateDefaultJobPartHeader( JobPartDeleteSnapshotsOption deleteSnapshotsOption = DefaultDeleteSnapshotsOption, JobPartPermanentDeleteOption permanentDeleteOption = DefaultPermanentDeleteOption, JobPartPlanRehydratePriorityType rehydratePriorityType = DefaultRehydratePriorityType, - DataTransferStatus atomicJobStatus = DefaultJobStatus, - DataTransferStatus atomicPartStatus = DefaultPartStatus) + DataTransferStatus atomicJobStatus = default, + DataTransferStatus atomicPartStatus = default) { if (startTime == default) { @@ -116,6 +116,8 @@ internal static JobPartPlanHeader CreateDefaultJobPartHeader( } metadata ??= DataProvider.BuildMetadata(); blobTags ??= DataProvider.BuildTags(); + atomicJobStatus ??= DefaultJobStatus; + atomicPartStatus ??= DefaultPartStatus; JobPartPlanDestinationBlob dstBlobData = new JobPartPlanDestinationBlob( blobType: blobType, diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferDownloadDirectoryTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferDownloadDirectoryTests.cs index 2dd38580978f..b122bdd77843 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferDownloadDirectoryTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferDownloadDirectoryTests.cs @@ -77,7 +77,7 @@ private async Task DownloadBlobDirectoryAndVerify( await testEventsRaised.AssertContainerCompletedCheck(sourceFiles.Count); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.Completed, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); // List all files in source blob folder path List blobNames = new List(); @@ -223,7 +223,7 @@ public async Task DownloadDirectoryAsync_Empty() await transfer.WaitForCompletionAsync(cancellationTokenSource.Token); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.Completed, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); List localItemsAfterDownload = Directory.GetFiles(destinationFolder, "*", SearchOption.AllDirectories).ToList(); @@ -517,7 +517,7 @@ public async Task StartTransfer_AwaitCompletion() // Assert Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.Completed, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); await testEventsRaised.AssertContainerCompletedCheck(4); } @@ -553,7 +553,8 @@ public async Task StartTransfer_AwaitCompletion_Failed() // Assert Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithFailedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); Assert.IsTrue(testEventsRaised.FailedEvents.First().Exception.Message.Contains("Cannot overwrite file.")); await testEventsRaised.AssertContainerCompletedWithFailedCheck(1); } @@ -591,7 +592,8 @@ public async Task StartTransfer_AwaitCompletion_Skipped() // Assert Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithSkippedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasSkippedItems); await testEventsRaised.AssertContainerCompletedWithSkippedCheck(1); } @@ -621,7 +623,7 @@ public async Task StartTransfer_EnsureCompleted() // Assert Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.Completed, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); await testEventsRaised.AssertContainerCompletedCheck(4); } @@ -657,7 +659,8 @@ public async Task StartTransfer_EnsureCompleted_Failed() // Assert Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithFailedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); Assert.IsTrue(testEventsRaised.FailedEvents.First().Exception.Message.Contains("Cannot overwrite file.")); await testEventsRaised.AssertContainerCompletedWithFailedCheck(1); } @@ -695,7 +698,8 @@ public async Task StartTransfer_EnsureCompleted_Skipped() // Assert Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithSkippedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasSkippedItems); await testEventsRaised.AssertContainerCompletedWithSkippedCheck(1); } @@ -734,7 +738,8 @@ public async Task StartTransfer_EnsureCompleted_Failed_SmallChunks() // Assert Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithFailedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); Assert.IsTrue(testEventsRaised.FailedEvents.First().Exception.Message.Contains("Cannot overwrite file.")); await testEventsRaised.AssertContainerCompletedWithFailedCheck(1); } diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferDownloadTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferDownloadTests.cs index b8c63c634e44..0d42188150cf 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferDownloadTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferDownloadTests.cs @@ -120,7 +120,7 @@ private async Task DownloadBlockBlobsAndVerify( options[i].TransferStatusChanged += (TransferStatusEventArgs args) => { // Assert - if (args.StorageTransferStatus == DataTransferStatus.Completed) + if (args.TransferStatus.HasCompletedSuccessfully) { completed = true; } @@ -172,7 +172,7 @@ private async Task DownloadBlockBlobsAndVerify( // Verify Download await downloadedBlobInfo[i].EventsRaised.AssertSingleCompletedCheck(); - Assert.AreEqual(DataTransferStatus.Completed, downloadedBlobInfo[i].DataTransfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, downloadedBlobInfo[i].DataTransfer.TransferStatus.State); CheckDownloadFile(downloadedBlobInfo[i].SourceLocalPath, downloadedBlobInfo[i].DestinationLocalPath); }; } @@ -288,7 +288,8 @@ public async Task BlockBlobToLocal_Skip_Exists() // Assert Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithSkippedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); Assert.IsTrue(skippedSeen); FileInfo destFileInfo = new FileInfo(destFile); Assert.IsTrue(destFileInfo.Length == 0); @@ -330,7 +331,8 @@ public async Task BlockBlobToLocal_Failure_Exists() // Assert Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithFailedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); await testEventsRaised.AssertSingleFailedCheck(); FileInfo destFileInfo = new FileInfo(destFile); Assert.IsTrue(destFileInfo.Length == 0); @@ -695,7 +697,8 @@ public async Task AppendBlobToLocal_Skip_Exists() // Assert Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithSkippedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasSkippedItems); await testEventsRaised.AssertSingleSkippedCheck(); FileInfo destFileInfo = new FileInfo(destFile); Assert.IsTrue(destFileInfo.Length == 0); @@ -738,7 +741,8 @@ public async Task AppendBlobToLocal_Failure_Exists() // Assert Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithFailedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); await testEventsRaised.AssertSingleFailedCheck(); FileInfo destFileInfo = new FileInfo(destFile); Assert.IsTrue(destFileInfo.Length == 0); @@ -1154,7 +1158,8 @@ public async Task PageBlobToLocal_Skip_Exists() // Assert Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithSkippedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasSkippedItems); await testEventsRaised.AssertSingleSkippedCheck(); FileInfo destFileInfo = new FileInfo(destFile); Assert.IsTrue(destFileInfo.Length == 0); @@ -1197,7 +1202,8 @@ public async Task PageBlobToLocal_Failure_Exists() // Assert Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithFailedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); await testEventsRaised.AssertSingleFailedCheck(); FileInfo destFileInfo = new FileInfo(destFile); Assert.IsTrue(destFileInfo.Length == 0); @@ -1417,7 +1423,7 @@ public async Task StartTransfer_AwaitCompletion() failureTransferHolder.AssertUnexpectedFailureCheck(); Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.Completed, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); } [RecordedTest] @@ -1448,7 +1454,8 @@ public async Task StartTransfer_AwaitCompletion_Failed() // Assert Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithFailedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); await testEventRaised.AssertSingleFailedCheck(); Assert.AreEqual(1, testEventRaised.FailedEvents.Count); Assert.IsTrue(testEventRaised.FailedEvents.First().Exception.Message.Contains("Cannot overwrite file.")); @@ -1483,7 +1490,8 @@ public async Task StartTransfer_AwaitCompletion_Skipped() // Assert Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithSkippedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasSkippedItems); await testEventRaised.AssertSingleSkippedCheck(); } @@ -1516,7 +1524,7 @@ public async Task StartTransfer_EnsureCompleted() testEventRaised.AssertUnexpectedFailureCheck(); Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.Completed, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); } [RecordedTest] @@ -1547,7 +1555,8 @@ public async Task StartTransfer_EnsureCompleted_Failed() // Assert Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithFailedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); await testEventsRaised.AssertSingleFailedCheck(); Assert.IsTrue(testEventsRaised.FailedEvents.First().Exception.Message.Contains("Cannot overwrite file.")); } @@ -1582,7 +1591,8 @@ public async Task StartTransfer_EnsureCompleted_Skipped() await testEventsRaised.AssertSingleSkippedCheck(); Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithSkippedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasSkippedItems); } #endregion } diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferSyncCopyDirectoryTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferSyncCopyDirectoryTests.cs index 7eff6aca930f..c14e6686f9d0 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferSyncCopyDirectoryTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferSyncCopyDirectoryTests.cs @@ -80,7 +80,7 @@ private async Task CopyBlobDirectoryAndVerify( await testEventFailed.AssertContainerCompletedCheck(sourceFiles.Count); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.Completed, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); // List all files in source blob folder path List sourceblobNames = new List(); @@ -247,7 +247,7 @@ public async Task BlockBlobDirectoryToDirectory_EmptyFolder() CancellationTokenSource tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); await transfer.WaitForCompletionAsync(tokenSource.Token); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.Completed, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); // Assert List blobs = ((List)await test.Container.GetBlobsAsync().ToListAsync()) @@ -588,7 +588,7 @@ public async Task StartTransfer_AwaitCompletion() testEventsRaised.AssertUnexpectedFailureCheck(); Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.Completed, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); } [Test] @@ -618,7 +618,8 @@ public async Task StartTransfer_AwaitCompletion_Failed() // Assert Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithFailedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); await testEventsRaised.AssertContainerCompletedWithFailedCheck(1); Assert.IsTrue(testEventsRaised.FailedEvents.First().Exception.Message.Contains("BlobAlreadyExists")); } @@ -651,7 +652,8 @@ public async Task StartTransfer_AwaitCompletion_Skipped() // Assert Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithSkippedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasSkippedItems); await testEventsRaised.AssertContainerCompletedWithSkippedCheck(1); } @@ -676,7 +678,7 @@ public async Task StartTransfer_EnsureCompleted() testEventsRaised.AssertUnexpectedFailureCheck(); Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.Completed, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); } [Test] @@ -706,7 +708,8 @@ public async Task StartTransfer_EnsureCompleted_Failed() // Assert Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithFailedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); await testEventsRaised.AssertContainerCompletedWithFailedCheck(1); Assert.IsTrue(testEventsRaised.FailedEvents.First().Exception.Message.Contains("BlobAlreadyExists")); } @@ -740,7 +743,8 @@ public async Task StartTransfer_EnsureCompleted_Skipped() testEventsRaised.AssertUnexpectedFailureCheck(); Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithSkippedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasSkippedItems); } [Test] @@ -775,7 +779,8 @@ public async Task StartTransfer_EnsureCompleted_Failed_SmallChunks() // Assert Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithFailedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); Assert.IsTrue(testEventsRaised.FailedEvents.First().Exception.Message.Contains("BlobAlreadyExists")); await testEventsRaised.AssertContainerCompletedWithFailedCheck(1); } diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferSyncCopyTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferSyncCopyTests.cs index 83e96dd93dcb..44f3c8ec0959 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferSyncCopyTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferSyncCopyTests.cs @@ -167,7 +167,7 @@ private async Task CopyBlockBlobsAndVerify( CancellationTokenSource tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(waitTimeInSec)); await copyBlobInfo[i].DataTransfer.WaitForCompletionAsync(tokenSource.Token); Assert.IsTrue(copyBlobInfo[i].DataTransfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.Completed, copyBlobInfo[i].DataTransfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, copyBlobInfo[i].DataTransfer.TransferStatus.State); // Verify Copy - using original source File and Copying the destination await copyBlobInfo[i].testEventsRaised.AssertSingleCompletedCheck(); @@ -387,7 +387,8 @@ public async Task BlockBlobToBlockBlob_Skip_Exists() // Assert Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithSkippedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasSkippedItems); await testEventsRaised.AssertSingleSkippedCheck(); Assert.IsTrue(await destinationClient.ExistsAsync()); // Verify Upload - That we skipped over and didn't reupload something new. @@ -439,7 +440,8 @@ public async Task BlockBlobToBlockBlob_Failure_Exists() // Assert Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithFailedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); Assert.IsTrue(await destinationClient.ExistsAsync()); await testEventsRaised.AssertSingleFailedCheck(); Assert.NotNull(testEventsRaised.FailedEvents.First().Exception, "Excepted failure: Overwrite failure was supposed to be raised during the test"); @@ -528,7 +530,8 @@ public async Task AppendBlobToAppendBlob_Error() // Assert Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithFailedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); await testEventsRaised.AssertSingleFailedCheck(); Assert.NotNull(testEventsRaised.FailedEvents.First().Exception, "Excepted failure: Failure was supposed to be raised during the test"); Assert.IsTrue(testEventsRaised.FailedEvents.First().Exception.Message.Contains("BlobAlreadyExists")); @@ -581,7 +584,8 @@ public async Task PageBlobToPageBlob_Error() // Assert Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithFailedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); await testEventsRaised.AssertSingleFailedCheck(); Assert.NotNull(testEventsRaised.FailedEvents.First().Exception, "Excepted failure: Failure was supposed to be raised during the test"); Assert.IsTrue(testEventsRaised.FailedEvents.First().Exception.Message.Contains("BlobAlreadyExists")); @@ -651,7 +655,7 @@ public async Task StartTransfer_AwaitCompletion() await testEventsRaised.AssertSingleCompletedCheck(); Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.Completed, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); } [RecordedTest] @@ -680,7 +684,8 @@ public async Task StartTransfer_AwaitCompletion_Failed() // Assert Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithFailedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); await testEventsRaised.AssertSingleFailedCheck(); Assert.IsTrue(testEventsRaised.FailedEvents.First().Exception.Message.Contains("BlobAlreadyExists")); } @@ -713,7 +718,8 @@ public async Task StartTransfer_AwaitCompletion_Skipped() await testEventsRaised.AssertSingleSkippedCheck(); Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithSkippedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasSkippedItems); } [RecordedTest] @@ -736,7 +742,7 @@ public async Task StartTransfer_EnsureCompleted() await testEventsRaised.AssertSingleCompletedCheck(); Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.Completed, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); } [RecordedTest] @@ -766,7 +772,8 @@ public async Task StartTransfer_EnsureCompleted_Failed() await testEventsRaised.AssertSingleFailedCheck(); Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithFailedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); Assert.IsTrue(testEventsRaised.FailedEvents.First().Exception.Message.Contains("BlobAlreadyExists")); } @@ -798,7 +805,8 @@ public async Task StartTransfer_EnsureCompleted_Skipped() await testEventsRaised.AssertSingleSkippedCheck(); Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithSkippedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasSkippedItems); } #endregion } diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferUploadDirectoryTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferUploadDirectoryTests.cs index aab0c62a2ab8..5862d00dcbbf 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferUploadDirectoryTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferUploadDirectoryTests.cs @@ -73,7 +73,7 @@ private async Task UploadBlobDirectoryAndVerify( await testEventsRaised.AssertContainerCompletedCheck(files.Count); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.Completed, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); // Assert - Check Response List blobs = ((List)await destinationContainer.GetBlobsAsync(prefix: destinationPrefix).ToListAsync()) @@ -736,7 +736,7 @@ public async Task StartTransfer_AwaitCompletion() // Assert Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.Completed, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); await testEventsRaised.AssertContainerCompletedCheck(4); } @@ -770,7 +770,8 @@ public async Task StartTransfer_AwaitCompletion_Failed() await testEventsRaised.AssertContainerCompletedWithFailedCheck(1); Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithFailedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); Assert.IsTrue(testEventsRaised.FailedEvents.First().Exception.Message.Contains("BlobAlreadyExists")); } @@ -805,7 +806,8 @@ public async Task StartTransfer_AwaitCompletion_Skipped() await testEventsRaised.AssertContainerCompletedWithSkippedCheck(1); Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithSkippedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasSkippedItems); } [Test] @@ -834,7 +836,7 @@ public async Task StartTransfer_EnsureCompleted() await testEventsRaised.AssertContainerCompletedCheck(4); Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.Completed, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); } [Test] @@ -867,7 +869,8 @@ public async Task StartTransfer_EnsureCompleted_Failed() await testEventsRaised.AssertContainerCompletedWithFailedCheck(1); Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithFailedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); Assert.IsTrue(testEventsRaised.FailedEvents.First().Exception.Message.Contains("BlobAlreadyExists")); } @@ -902,7 +905,8 @@ public async Task StartTransfer_EnsureCompleted_Skipped() await testEventsRaised.AssertContainerCompletedWithSkippedCheck(1); Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithSkippedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasSkippedItems); } [Test] @@ -937,7 +941,8 @@ public async Task StartTransfer_EnsureCompleted_Failed_SmallChunks() // Assert Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithFailedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); Assert.IsTrue(testEventsRaised.FailedEvents.First().Exception.Message.Contains("BlobAlreadyExists")); await testEventsRaised.AssertContainerCompletedWithFailedCheck(1); } diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferUploadTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferUploadTests.cs index bf8e8c2c2c0d..f70725104019 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferUploadTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferUploadTests.cs @@ -169,7 +169,7 @@ public async Task LocalToBlockBlob_EventHandler() options.TransferStatusChanged += (TransferStatusEventArgs args) => { // Assert - if (args.StorageTransferStatus == DataTransferStatus.InProgress) + if (args.TransferStatus.State == DataTransferState.InProgress) { progressSeen = true; } @@ -305,7 +305,8 @@ public async Task LocalToBlockBlob_Skip_Exists() await testEventsRaised.AssertSingleSkippedCheck(); Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithSkippedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasSkippedItems); Assert.IsTrue(await destinationClient.ExistsAsync()); // Verify Upload - That we skipped over and didn't reupload something new. using (FileStream fileStream = File.OpenRead(originalSourceFile)) @@ -352,7 +353,8 @@ public async Task LocalToBlockBlob_Failure_Exists() // Assert Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithFailedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); Assert.IsTrue(await destinationClient.ExistsAsync()); await testEventRaised.AssertSingleFailedCheck(); Assert.NotNull(testEventRaised.FailedEvents.First().Exception, "Excepted failure: Overwrite failure was supposed to be raised during the test"); @@ -581,7 +583,7 @@ private async Task UploadPageBlobsAndVerify( CancellationTokenSource tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(waitTimeInSec)); await uploadedBlobInfo[i].DataTransfer.WaitForCompletionAsync(tokenSource.Token); Assert.IsTrue(uploadedBlobInfo[i].DataTransfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.Completed, uploadedBlobInfo[i].DataTransfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, uploadedBlobInfo[i].DataTransfer.TransferStatus.State); // Verify Upload await uploadedBlobInfo[i].EventsRaised.AssertSingleCompletedCheck(); @@ -695,7 +697,8 @@ public async Task LocalToPageBlob_Skip_Exists() await testEventsRaised.AssertSingleSkippedCheck(); Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithSkippedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasSkippedItems); Assert.IsTrue(await destinationClient.ExistsAsync()); // Verify Upload - That we skipped over and didn't reupload something new. using (FileStream fileStream = File.OpenRead(originalSourceFile)) @@ -742,7 +745,8 @@ public async Task LocalToPageBlob_Failure_Exists() await testEventsRaised.AssertSingleFailedCheck(); Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithFailedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); Assert.IsTrue(await destinationClient.ExistsAsync()); Assert.NotNull(testEventsRaised.FailedEvents.First().Exception, "Excepted failure: Overwrite failure was supposed to be raised during the test"); Assert.IsTrue(testEventsRaised.FailedEvents.First().Exception.Message.Contains("The specified blob already exists.")); @@ -1000,7 +1004,7 @@ private async Task UploadAppendBlobsAndVerify( CancellationTokenSource tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(waitTimeInSec)); await uploadedBlobInfo[i].DataTransfer.WaitForCompletionAsync(tokenSource.Token); Assert.IsTrue(uploadedBlobInfo[i].DataTransfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.Completed, uploadedBlobInfo[i].DataTransfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, uploadedBlobInfo[i].DataTransfer.TransferStatus.State); // Verify Upload await uploadedBlobInfo[i].EventsRaised.AssertSingleCompletedCheck(); @@ -1136,7 +1140,8 @@ public async Task LocalToAppendBlob_Skip_Exists() await testEventsRaised.AssertSingleSkippedCheck(); Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithSkippedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasSkippedItems); Assert.IsTrue(await destinationClient.ExistsAsync()); // Verify Upload - That we skipped over and didn't reupload something new. using (FileStream fileStream = File.OpenRead(originalSourceFile)) @@ -1183,7 +1188,8 @@ public async Task LocalToAppendBlob_Failure_Exists() await testEventsRaised.AssertSingleFailedCheck(); Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithFailedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); Assert.IsTrue(await destinationClient.ExistsAsync()); Assert.NotNull(testEventsRaised.FailedEvents.First().Exception, "Excepted failure: Overwrite failure was supposed to be raised during the test"); Assert.IsTrue(testEventsRaised.FailedEvents.First().Exception.Message.Contains("The specified blob already exists.")); @@ -1414,7 +1420,7 @@ public async Task StartTransfer_AwaitCompletion() await testEventsRaised.AssertSingleCompletedCheck(); Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.Completed, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); } [RecordedTest] @@ -1444,7 +1450,8 @@ public async Task StartTransfer_AwaitCompletion_Failed() await testEventsRaised.AssertSingleFailedCheck(); Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithFailedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); Assert.IsTrue(testEventsRaised.FailedEvents.First().Exception.Message.Contains("BlobAlreadyExists")); } @@ -1476,7 +1483,8 @@ public async Task StartTransfer_AwaitCompletion_Skipped() await testEventsRaised.AssertSingleSkippedCheck(); Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithSkippedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasSkippedItems); } [RecordedTest] @@ -1499,7 +1507,7 @@ public async Task StartTransfer_EnsureCompleted() await testEventsRaised.AssertSingleCompletedCheck(); Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.Completed, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); } [RecordedTest] @@ -1529,7 +1537,8 @@ public async Task StartTransfer_EnsureCompleted_Failed() await testEventsRaised.AssertSingleFailedCheck(); Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithFailedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); Assert.IsTrue(testEventsRaised.FailedEvents.First().Exception.Message.Contains("BlobAlreadyExists")); } @@ -1561,7 +1570,8 @@ public async Task StartTransfer_EnsureCompleted_Skipped() await testEventsRaised.AssertSingleSkippedCheck(); Assert.NotNull(transfer); Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferStatus.CompletedWithSkippedTransfers, transfer.TransferStatus); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasSkippedItems); } #endregion diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/TestEventsRaised.cs b/sdk/storage/Azure.Storage.DataMovement/tests/TestEventsRaised.cs index e6cd6c2ba449..608381172706 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/TestEventsRaised.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/TestEventsRaised.cs @@ -24,6 +24,14 @@ namespace Azure.Storage.DataMovement.Tests /// internal class TestEventsRaised : IDisposable { + private static readonly DataTransferStatus InProgressStatus = new DataTransferStatus(DataTransferState.InProgress, false, false); + private static readonly DataTransferStatus InProgressFailedStatus = new DataTransferStatus(DataTransferState.InProgress, true, false); + private static readonly DataTransferStatus InProgressSkippedStatus = new DataTransferStatus(DataTransferState.InProgress, false, true); + private static readonly DataTransferStatus StoppingFailedStatus = new DataTransferStatus(DataTransferState.Stopping, true, false); + private static readonly DataTransferStatus SuccessfulCompletedStatus = new DataTransferStatus(DataTransferState.Completed, false, false); + private static readonly DataTransferStatus SkippedCompletedStatus = new DataTransferStatus(DataTransferState.Completed, false, true); + private static readonly DataTransferStatus FailedCompletedStatus = new DataTransferStatus(DataTransferState.Completed, true, false); + public List FailedEvents { get; internal set; } public List StatusEvents { get; internal set; } public List SkippedEvents { get; internal set; } @@ -98,6 +106,25 @@ private Task AppendSingleTransferCompleted(TransferItemCompletedEventArgs args) return Task.CompletedTask; } + private void AssertTransferStatusCollection(DataTransferStatus[] expected, DataTransferStatus[] actual) + { + Assert.AreEqual(expected.Length, actual.Length); + Assert.Multiple(() => + { + for (int i = 0; i < expected.Length; i++) + { + if (!expected[i].Equals(actual[i])) + { + // Compared to individual Assert statements, this makes it clear when running the test, what's failing. + Assert.Fail($"Transfer State Mismatch:" + + $"Expected {nameof(DataTransferStatus.State)}: {expected[i].State}; Actual {nameof(DataTransferStatus.State)}: {actual[i].State}\n" + + $"Expected {nameof(DataTransferStatus.HasFailedItems)}: {expected[i].HasFailedItems}; Actual {nameof(DataTransferStatus.HasFailedItems)}: {actual[i].HasFailedItems}\n" + + $"Expected {nameof(DataTransferStatus.HasSkippedItems)}: {expected[i].HasSkippedItems}; Actual {nameof(DataTransferStatus.HasSkippedItems)}: {actual[i].HasSkippedItems}\n"); + } + } + }); + } + public void AssertUnexpectedFailureCheck() { Assert.Multiple(() => @@ -116,7 +143,7 @@ public void AssertUnexpectedFailureCheck() /// /// This asserts that the expected events occurred during a single transfer that is expected - /// to have a at the end without any skips + /// to have a at the end without any skips /// or failures. /// public async Task AssertSingleCompletedCheck() @@ -126,16 +153,16 @@ public async Task AssertSingleCompletedCheck() Assert.IsEmpty(SingleCompletedEvents); await WaitForStatusEventsAsync().ConfigureAwait(false); - CollectionAssert.AreEqual( + AssertTransferStatusCollection( new DataTransferStatus[] { - DataTransferStatus.InProgress, - DataTransferStatus.Completed }, - StatusEvents.Select(e => e.StorageTransferStatus)); + InProgressStatus, + SuccessfulCompletedStatus }, + StatusEvents.Select(e => e.TransferStatus).ToArray()); } /// /// This asserts that the expected events occurred during a single transfer that is expected - /// to have a at the end without any + /// to have a at the end without any /// or failures. /// public async Task AssertSingleSkippedCheck() @@ -147,16 +174,17 @@ public async Task AssertSingleSkippedCheck() Assert.NotNull(SkippedEvents.First().DestinationResource.Uri); await WaitForStatusEventsAsync().ConfigureAwait(false); - CollectionAssert.AreEqual( + AssertTransferStatusCollection( new DataTransferStatus[] { - DataTransferStatus.InProgress, - DataTransferStatus.CompletedWithSkippedTransfers }, - StatusEvents.Select(e => e.StorageTransferStatus)); + InProgressStatus, + InProgressSkippedStatus, + SkippedCompletedStatus }, + StatusEvents.Select(e => e.TransferStatus).ToArray()); } /// /// This asserts that the expected events occurred during a single transfer that is expected - /// to have a at the end without any skips. + /// to have a at the end without any skips. /// public async Task AssertSingleFailedCheck() { @@ -168,17 +196,18 @@ public async Task AssertSingleFailedCheck() Assert.NotNull(FailedEvents.First().DestinationResource.Uri); await WaitForStatusEventsAsync().ConfigureAwait(false); - CollectionAssert.AreEqual( + AssertTransferStatusCollection( new DataTransferStatus[] { - DataTransferStatus.InProgress, - DataTransferStatus.CancellationInProgress, - DataTransferStatus.CompletedWithFailedTransfers }, - StatusEvents.Select(e => e.StorageTransferStatus)); + InProgressStatus, + InProgressFailedStatus, + StoppingFailedStatus, + FailedCompletedStatus }, + StatusEvents.Select(e => e.TransferStatus).ToArray()); } /// /// This asserts that the expected events occurred during a container transfer that is expected - /// to have a at the end without any skips + /// to have a at the end without any skips /// or failures. /// /// @@ -191,16 +220,16 @@ public async Task AssertContainerCompletedCheck(int transferCount) Assert.AreEqual(transferCount, SingleCompletedEvents.Count); await WaitForStatusEventsAsync().ConfigureAwait(false); - CollectionAssert.AreEqual( + AssertTransferStatusCollection( new DataTransferStatus[] { - DataTransferStatus.InProgress, - DataTransferStatus.Completed }, - StatusEvents.Select(e => e.StorageTransferStatus)); + InProgressStatus, + SuccessfulCompletedStatus }, + StatusEvents.Select(e => e.TransferStatus).ToArray()); } /// /// This asserts that the expected events occurred during a container transfer that is expected - /// to have a at the end without any skips. + /// to have a at the end without any skips. /// Assuming was set. /// /// @@ -221,17 +250,18 @@ public async Task AssertContainerCompletedWithFailedCheck(int expectedFailureCou Assert.IsEmpty(SkippedEvents); await WaitForStatusEventsAsync().ConfigureAwait(false); - CollectionAssert.AreEqual( + AssertTransferStatusCollection( new DataTransferStatus[] { - DataTransferStatus.InProgress, - DataTransferStatus.CancellationInProgress, - DataTransferStatus.CompletedWithFailedTransfers }, - StatusEvents.Select(e => e.StorageTransferStatus)); + InProgressStatus, + InProgressFailedStatus, + StoppingFailedStatus, + FailedCompletedStatus }, + StatusEvents.Select(e => e.TransferStatus).ToArray()); } /// /// This asserts that the expected events occurred during a container transfer that is expected - /// to have a at the end without any skips. + /// to have a at the end without any skips. /// Assuming was set. /// /// @@ -252,16 +282,17 @@ public async Task AssertContainerCompletedWithFailedCheckContinue(int expectedFa Assert.IsEmpty(SkippedEvents); await WaitForStatusEventsAsync().ConfigureAwait(false); - CollectionAssert.AreEqual( + AssertTransferStatusCollection( new DataTransferStatus[] { - DataTransferStatus.InProgress, - DataTransferStatus.CompletedWithFailedTransfers }, - StatusEvents.Select(e => e.StorageTransferStatus)); + InProgressStatus, + InProgressFailedStatus, + FailedCompletedStatus }, + StatusEvents.Select(e => e.TransferStatus).ToArray()); } /// /// This asserts that the expected events occurred during a container transfer that is expected - /// to have a at the end without any failures. + /// to have a at the end without any failures. /// /// /// Expected amount of skipped single transfers to occur within the container transfers. @@ -272,11 +303,12 @@ public async Task AssertContainerCompletedWithSkippedCheck(int expectedSkipCount Assert.AreEqual(expectedSkipCount, SkippedEvents.Count); await WaitForStatusEventsAsync().ConfigureAwait(false); - CollectionAssert.AreEqual( + AssertTransferStatusCollection( new DataTransferStatus[] { - DataTransferStatus.InProgress, - DataTransferStatus.CompletedWithSkippedTransfers }, - StatusEvents.Select(e => e.StorageTransferStatus)); + InProgressStatus, + InProgressSkippedStatus, + SkippedCompletedStatus }, + StatusEvents.Select(e => e.TransferStatus).ToArray()); } public async Task AssertPausedCheck() @@ -285,11 +317,11 @@ public async Task AssertPausedCheck() Assert.IsEmpty(SkippedEvents); await WaitForStatusEventsAsync().ConfigureAwait(false); - CollectionAssert.AreEqual( + AssertTransferStatusCollection( new DataTransferStatus[] { - DataTransferStatus.InProgress, - DataTransferStatus.Paused }, - StatusEvents.Select(e => e.StorageTransferStatus)); + InProgressStatus, + new DataTransferStatus(DataTransferState.Paused, false, false) }, + StatusEvents.Select(e => e.TransferStatus).ToArray()); } ///