[Storage] Added Fix for Creating Share File before service copy; Added Data Movement E2E service copy tests

sdk/storage/Azure.Storage.DataMovement.Files.Shares/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "net",
   "TagPrefix": "net/storage/Azure.Storage.DataMovement.Files.Shares",
-  "Tag": "net/storage/Azure.Storage.DataMovement.Files.Shares_29f88351cb"
+  "Tag": "net/storage/Azure.Storage.DataMovement.Files.Shares_02f417687f"
 }

sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs

@@ -158,12 +158,16 @@ protected override async Task CopyFromUriAsync(
             CancellationToken cancellationToken = default)
         {
             CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
-            await ShareFileClient.UploadRangeFromUriAsync(
-                sourceUri: sourceResource.Uri,
-                range: new HttpRange(0, completeLength),
-                sourceRange: new HttpRange(0, completeLength),
-                options: _options.ToShareFileUploadRangeFromUriOptions(),
-                cancellationToken: cancellationToken).ConfigureAwait(false);
+            await CreateAsync(overwrite, completeLength, cancellationToken).ConfigureAwait(false);
+            if (completeLength > 0)
+            {
+                await ShareFileClient.UploadRangeFromUriAsync(
+                    sourceUri: sourceResource.Uri,
+                    range: new HttpRange(0, completeLength),
+                    sourceRange: new HttpRange(0, completeLength),
+                    options: _options.ToShareFileUploadRangeFromUriOptions(),
+                    cancellationToken: cancellationToken).ConfigureAwait(false);
+            }
         }
 
         protected override async Task<bool> DeleteIfExistsAsync(CancellationToken cancellationToken = default)

sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj

@@ -44,6 +44,7 @@
     <Compile Include="$(AzureStorageDataMovementTestSharedSources)TestEventsRaised.cs" LinkBase="Shared\DataMovement" />
     <Compile Include="$(AzureStorageDataMovementTestSharedSources)DisposingLocalDirectory.cs" LinkBase="Shared\DataMovement" />
     <Compile Include="$(AzureStorageDataMovementTestSharedSources)StartTransferUploadTestBase.cs" LinkBase="Shared\DataMovement" />
+    <Compile Include="$(AzureStorageDataMovementTestSharedSources)StartTransferCopyTestBase.cs" LinkBase="Shared\DataMovement" />
     <Compile Include="$(AzureStorageDataMovementTestSharedSources)StartTransferDownloadTestBase.cs" LinkBase="Shared\DataMovement" />
   </ItemGroup>
   <ItemGroup>

sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ClientBuilderExtensions.cs

@@ -10,6 +10,7 @@
 using SharesClientBuilder = Azure.Storage.Test.Shared.ClientBuilder<
     Azure.Storage.Files.Shares.ShareServiceClient,
     Azure.Storage.Files.Shares.ShareClientOptions>;
+using Azure.Storage.Files.Shares.Models;
 
 namespace Azure.Storage.DataMovement.Files.Shares.Tests
 {
@@ -42,6 +43,17 @@ public static SharesClientBuilder GetNewShareClientBuilder(TenantConfigurationBu
                 (uri, azureSasCredential, clientOptions) => new ShareServiceClient(uri, azureSasCredential, clientOptions),
                 () => new ShareClientOptions(serviceVersion));
 
+        public static ShareServiceClient GetServiceClient_OAuthAccount_SharedKey(this SharesClientBuilder clientBuilder) =>
+            clientBuilder.GetServiceClientFromSharedKeyConfig(clientBuilder.Tenants.TestConfigOAuth);
+
+        public static ShareServiceClient GetServiceClient_OAuth(
+            this SharesClientBuilder clientBuilder, ShareClientOptions options = default)
+        {
+            options ??= clientBuilder.GetOptions();
+            options.ShareTokenIntent = ShareTokenIntent.Backup;
+            return clientBuilder.GetServiceClientFromOauthConfig(clientBuilder.Tenants.TestConfigOAuth, options);
+        }
+
         public static async Task<DisposingShare> GetTestShareAsync(
             this SharesClientBuilder clientBuilder,
             ShareServiceClient service = default,

sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareFileResourceTests.cs

@@ -313,6 +313,22 @@ public async Task CopyFromUriAsync()
                         contentHash: default,
                         isServerEncrypted: false),
                     new MockResponse(200))));
+            mockDestination.Setup(b => b.ExistsAsync(It.IsAny<CancellationToken>()))
+                .Returns(Task.FromResult(Response.FromValue(false, new MockResponse(200))));
+            mockDestination.Setup(b => b.CreateAsync(It.IsAny<long>(), It.IsAny<ShareFileHttpHeaders>(), It.IsAny<Dictionary<string, string>>(), It.IsAny<FileSmbProperties>(), It.IsAny<string>(), It.IsAny<ShareFileRequestConditions>(), It.IsAny<CancellationToken>()))
+                .Returns(Task.FromResult(Response.FromValue(
+                    FilesModelFactory.StorageFileInfo(
+                        eTag: new ETag("eTag"),
+                        lastModified: DateTimeOffset.UtcNow,
+                        isServerEncrypted: false,
+                        filePermissionKey: "rw",
+                        fileAttributes: "Archive|ReadOnly",
+                        fileCreationTime: DateTimeOffset.UtcNow,
+                        fileLastWriteTime: DateTimeOffset.UtcNow,
+                        fileChangeTime: DateTimeOffset.UtcNow,
+                        fileId: "48903841",
+                        fileParentId: "93024923"),
+                    new MockResponse(200))));
             ShareFileStorageResource destinationResource = new ShareFileStorageResource(mockDestination.Object);
 
             int length = 1024;
@@ -327,6 +343,18 @@ public async Task CopyFromUriAsync()
                 It.IsAny<ShareFileUploadRangeFromUriOptions>(),
                 It.IsAny<CancellationToken>()),
                 Times.Once());
+            mockDestination.Verify(b => b.CreateAsync(
+                length,
+                It.IsAny<ShareFileHttpHeaders>(),
+                It.IsAny<Dictionary<string, string>>(),
+                It.IsAny<FileSmbProperties>(),
+                It.IsAny<string>(),
+                It.IsAny<ShareFileRequestConditions>(),
+                It.IsAny<CancellationToken>()),
+                Times.Once());
+            mockDestination.Verify(b => b.ExistsAsync(
+                It.IsAny<CancellationToken>()),
+                Times.Once());
             mockDestination.VerifyNoOtherCalls();
         }
 
@@ -342,28 +370,24 @@ public async Task CopyFromUriAsync_Error()
                 new Uri("https://storageaccount.file.core.windows.net/container/destinationfile"),
                 new ShareClientOptions());
 
-            mockDestination.Setup(b => b.UploadRangeFromUriAsync(It.IsAny<Uri>(), It.IsAny<HttpRange>(), It.IsAny<HttpRange>(), It.IsAny<ShareFileUploadRangeFromUriOptions>(), It.IsAny<CancellationToken>()))
-                .Throws(new RequestFailedException(status: 404, message: "The specified resource does not exist.", errorCode: "ResourceNotFound", default));
+            mockDestination.Setup(b => b.ExistsAsync(It.IsAny<CancellationToken>()))
+                .Returns(Task.FromResult(Response.FromValue(true, new MockResponse(200))));
             ShareFileStorageResource destinationResource = new ShareFileStorageResource(mockDestination.Object);
 
             // Act
             int length = 1024;
-            await TestHelper.AssertExpectedExceptionAsync<RequestFailedException>(
-                destinationResource.CopyFromUriInternalAsync(sourceResource.Object, false, length),
+            await TestHelper.AssertExpectedExceptionAsync<InvalidOperationException>(
+                destinationResource.CopyBlockFromUriInternalAsync(sourceResource.Object, new HttpRange(0, length), false, length),
                 e =>
                 {
-                    Assert.AreEqual("ResourceNotFound", e.ErrorCode);
+                    Assert.IsTrue(e.Message.Contains("Cannot overwrite file."));
                 });
 
-            sourceResource.Verify(b => b.Uri, Times.Once());
             sourceResource.VerifyNoOtherCalls();
-            mockDestination.Verify(b => b.UploadRangeFromUriAsync(
-                sourceResource.Object.Uri,
-                new HttpRange(0, length),
-                new HttpRange(0, length),
-                It.IsAny<ShareFileUploadRangeFromUriOptions>(),
+            mockDestination.Verify(b => b.ExistsAsync(
                 It.IsAny<CancellationToken>()),
                 Times.Once());
+            mockDestination.Verify(b => b.Path, Times.Once());
             mockDestination.VerifyNoOtherCalls();
         }
 

sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareFileStartTransferCopyTests.cs

@@ -0,0 +1,142 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using System.Threading.Tasks;
+using Azure.Storage.Test.Shared;
+using Azure.Storage.DataMovement.Tests;
+using Azure.Storage.Files.Shares;
+using Azure.Storage.Files.Shares.Tests;
+using System.IO;
+using Azure.Core;
+using Azure.Core.TestFramework;
+
+namespace Azure.Storage.DataMovement.Files.Shares.Tests
+{
+    [ShareClientTestFixture]
+    public class ShareFileStartTransferCopyTests : StartTransferCopyTestBase
+        <ShareServiceClient,
+        ShareClient,
+        ShareFileClient,
+        ShareClientOptions,
+        ShareServiceClient,
+        ShareClient,
+        ShareFileClient,
+        ShareClientOptions,
+        StorageTestEnvironment>
+    {
+        private const string _fileResourcePrefix = "test-file-";
+        private const string _expectedOverwriteExceptionMessage = "Cannot overwrite file.";
+        protected readonly ShareClientOptions.ServiceVersion _serviceVersion;
+
+        public ShareFileStartTransferCopyTests(bool async, ShareClientOptions.ServiceVersion serviceVersion)
+            : base(async, _expectedOverwriteExceptionMessage, _fileResourcePrefix, null /* RecordedTestMode.Record /* to re-record */)
+        {
+            _serviceVersion = serviceVersion;
+            SourceClientBuilder = ClientBuilderExtensions.GetNewShareClientBuilder(Tenants, serviceVersion);
+            DestinationClientBuilder = ClientBuilderExtensions.GetNewShareClientBuilder(Tenants, serviceVersion);
+        }
+
+        protected override async Task<bool> SourceExistsAsync(ShareFileClient objectClient)
+            => await objectClient.ExistsAsync();
+
+        protected override async Task<bool> DestinationExistsAsync(ShareFileClient objectClient)
+            => await objectClient.ExistsAsync();
+
+        protected override async Task<IDisposingContainer<ShareClient>> GetSourceDisposingContainerAsync(ShareServiceClient service = null, string containerName = null)
+            => await SourceClientBuilder.GetTestShareAsync(service, containerName);
+
+        protected override async Task<IDisposingContainer<ShareClient>> GetDestinationDisposingContainerAsync(ShareServiceClient service = null, string containerName = null)
+            => await DestinationClientBuilder.GetTestShareAsync(service, containerName);
+
+        private async Task<ShareFileClient> CreateFileClientAsync(
+            ShareClient container,
+            long? objectLength = null,
+            bool createResource = false,
+            string objectName = null,
+            ShareClientOptions options = null,
+            Stream contents = null)
+        {
+            objectName ??= GetNewObjectName();
+            ShareFileClient fileClient = container.GetRootDirectoryClient().GetFileClient(objectName);
+            if (createResource)
+            {
+                if (!objectLength.HasValue)
+                {
+                    throw new InvalidOperationException($"Cannot create share file without size specified. Either set {nameof(createResource)} to false or specify a {nameof(objectLength)}.");
+                }
+                await fileClient.CreateAsync(objectLength.Value);
+
+                if (contents != default)
+                {
+                    await fileClient.UploadAsync(contents);
+                }
+            }
+            Uri sourceUri = fileClient.GenerateSasUri(Sas.ShareFileSasPermissions.All, Recording.UtcNow.AddDays(1));
+            return InstrumentClient(new ShareFileClient(sourceUri, GetOptions()));
+        }
+
+        protected override Task<ShareFileClient> GetSourceObjectClientAsync(
+            ShareClient container,
+            long? objectLength = null,
+            bool createResource = false,
+            string objectName = null,
+            ShareClientOptions options = null,
+            Stream contents = null)
+            => CreateFileClientAsync(
+                container,
+                objectLength,
+                createResource,
+                objectName,
+                options,
+                contents);
+
+        protected override StorageResourceItem GetSourceStorageResourceItem(ShareFileClient objectClient)
+            => new ShareFileStorageResource(objectClient);
+
+        protected override Task<Stream> SourceOpenReadAsync(ShareFileClient objectClient)
+            => objectClient.OpenReadAsync();
+
+        protected override Task<ShareFileClient> GetDestinationObjectClientAsync(
+            ShareClient container,
+            long? objectLength = null,
+            bool createResource = false,
+            string objectName = null,
+            ShareClientOptions options = null,
+            Stream contents = null)
+            => CreateFileClientAsync(
+                container,
+                objectLength,
+                createResource,
+                objectName,
+                options,
+                contents);
+
+        protected override StorageResourceItem GetDestinationStorageResourceItem(ShareFileClient objectClient)
+            => new ShareFileStorageResource(objectClient);
+
+        protected override Task<Stream> DestinationOpenReadAsync(ShareFileClient objectClient)
+            => objectClient.OpenReadAsync();
+
+        public ShareClientOptions GetOptions()
+        {
+            var options = new ShareClientOptions(_serviceVersion)
+            {
+                Diagnostics = { IsLoggingEnabled = true },
+                Retry =
+                {
+                    Mode = RetryMode.Exponential,
+                    MaxRetries = Constants.MaxReliabilityRetries,
+                    Delay = TimeSpan.FromSeconds(Mode == RecordedTestMode.Playback ? 0.01 : 1),
+                    MaxDelay = TimeSpan.FromSeconds(Mode == RecordedTestMode.Playback ? 0.1 : 60)
+                },
+            };
+            if (Mode != RecordedTestMode.Live)
+            {
+                options.AddPolicy(new RecordedClientRequestIdPolicy(Recording), HttpPipelinePosition.PerCall);
+            }
+
+            return InstrumentClientOptions(options);
+        }
+    }
+}

sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceJobPart.cs

@@ -210,6 +210,12 @@ await _destinationResource.CopyFromUriAsync(
             {
                 await InvokeSkippedArg().ConfigureAwait(false);
             }
+            catch (InvalidOperationException ex)
+            when (_createMode == StorageResourceCreationPreference.SkipIfExists
+                && ex.Message.Contains("Cannot overwrite file."))
+            {
+                await InvokeSkippedArg().ConfigureAwait(false);
+            }
             catch (Exception ex)
             {
                 await InvokeFailedArg(ex).ConfigureAwait(false);