diff --git a/src/Storage/Storage.Management.Test/ScenarioTests/StorageDataPlaneTests.ps1 b/src/Storage/Storage.Management.Test/ScenarioTests/StorageDataPlaneTests.ps1 index c7f302accab6..a666cb43c1d1 100644 --- a/src/Storage/Storage.Management.Test/ScenarioTests/StorageDataPlaneTests.ps1 +++ b/src/Storage/Storage.Management.Test/ScenarioTests/StorageDataPlaneTests.ps1 @@ -288,13 +288,17 @@ function Test-Blob # Encryption Scope Test $scopename = "testscope" + $scopename2 = "testscope2" $containerName2 = "testscopecontainer" New-AzStorageEncryptionScope -ResourceGroupName $ResourceGroupName -StorageAccountName $storageAccountName -EncryptionScopeName $scopename -StorageEncryption - $container = New-AzStorageContainer -Name $containerName2 -Context $storageContext -DefaultEncryptionScope $scopeName2 -PreventEncryptionScopeOverride $true + New-AzStorageEncryptionScope -ResourceGroupName $ResourceGroupName -StorageAccountName $storageAccountName -EncryptionScopeName $scopename2 -StorageEncryption + $container = New-AzStorageContainer -Name $containerName2 -Context $storageContext -DefaultEncryptionScope $scopeName -PreventEncryptionScopeOverride $true Assert-AreEqual $scopename $container.BlobContainerProperties.DefaultEncryptionScope Assert-AreEqual $true $container.BlobContainerProperties.PreventEncryptionScopeOverride $blob = Set-AzStorageBlobContent -Context $storageContext -File $localSrcFile -Container $containerName -Blob encryscopetest -EncryptionScope $scopename Assert-AreEqual $scopename $blob.BlobProperties.EncryptionScope + $blob = Copy-AzStorageBlob -Context $storageContext -SrcContainer $containerName -SrcBlob encryscopetest -DestContainer $containerName -DestBlob encryscopetest -Force -EncryptionScope $scopename2 + Assert-AreEqual $scopename2 $blob.BlobProperties.EncryptionScope Remove-AzStorageContainer -Name $containerName2 -Force -Context $storageContext # Clean Storage Account diff --git a/src/Storage/Storage.Management/Az.Storage.psd1 b/src/Storage/Storage.Management/Az.Storage.psd1 index 698c9b7d34c1..53613d111c04 100644 --- a/src/Storage/Storage.Management/Az.Storage.psd1 +++ b/src/Storage/Storage.Management/Az.Storage.psd1 @@ -179,7 +179,8 @@ CmdletsToExport = 'Get-AzStorageAccount', 'Get-AzStorageAccountKey', 'Set-AzDataLakeGen2AclRecursive', 'Update-AzDataLakeGen2AclRecursive', 'Remove-AzDataLakeGen2AclRecursive', 'New-AzStorageEncryptionScope', - 'Update-AzStorageEncryptionScope', 'Get-AzStorageEncryptionScope' + 'Update-AzStorageEncryptionScope', 'Get-AzStorageEncryptionScope', + 'Copy-AzStorageBlob' # Variables to export from this module # VariablesToExport = @() diff --git a/src/Storage/Storage.Management/ChangeLog.md b/src/Storage/Storage.Management/ChangeLog.md index d23e7d987259..b48ecacc3384 100644 --- a/src/Storage/Storage.Management/ChangeLog.md +++ b/src/Storage/Storage.Management/ChangeLog.md @@ -34,6 +34,8 @@ - `Set-AzStorageAccount` * Supported create Encryption Scope with RequireInfrastructureEncryption - `New-AzStorageEncryptionScope` +* Supported copy block blob synchronously, with encryption scope + - `Copy-AzStorageBlob` ## Version 3.3.0 * Supported RoutingPreference settings in create/update Storage account diff --git a/src/Storage/Storage.Management/help/Az.Storage.md b/src/Storage/Storage.Management/help/Az.Storage.md index db6cc2af42dc..d62f6b18cc44 100644 --- a/src/Storage/Storage.Management/help/Az.Storage.md +++ b/src/Storage/Storage.Management/help/Az.Storage.md @@ -23,6 +23,9 @@ Adds an action to the input ManagementPolicy Action Group object, or creates a M ### [Close-AzStorageFileHandle](Close-AzStorageFileHandle.md) Closes file handles of a file share, a file directory or a file. +### [Copy-AzStorageBlob](Copy-AzStorageBlob.md) +Copy a blob synchronously. + ### [Disable-AzStorageBlobDeleteRetentionPolicy](Disable-AzStorageBlobDeleteRetentionPolicy.md) Disable delete retention policy for the Azure Storage Blob service. diff --git a/src/Storage/Storage.Management/help/Copy-AzStorageBlob.md b/src/Storage/Storage.Management/help/Copy-AzStorageBlob.md new file mode 100644 index 000000000000..2a569d1660c9 --- /dev/null +++ b/src/Storage/Storage.Management/help/Copy-AzStorageBlob.md @@ -0,0 +1,379 @@ +--- +external help file: Microsoft.Azure.PowerShell.Cmdlets.Storage.dll-Help.xml +Module Name: Az.Storage +online version: https://docs.microsoft.com/powershell/module/az.storage/copy-azstorageblob +schema: 2.0.0 +--- + +# Copy-AzStorageBlob + +## SYNOPSIS +Copy a blob synchronously. + +## SYNTAX + +### ContainerName (Default) +``` +Copy-AzStorageBlob [-DestBlobType ] [-SrcBlob] -SrcContainer -DestContainer + [-DestBlob ] [-StandardBlobTier ] [-RehydratePriority ] + [-EncryptionScope ] [-Context ] [-DestContext ] [-Force] [-AsJob] + [-DefaultProfile ] [-WhatIf] [-Confirm] [] +``` + +### BlobInstance +``` +Copy-AzStorageBlob [-BlobBaseClient ] [-DestBlobType ] -DestContainer + [-DestBlob ] [-StandardBlobTier ] [-RehydratePriority ] + [-EncryptionScope ] [-Context ] [-DestContext ] [-Force] [-AsJob] + [-DefaultProfile ] [-WhatIf] [-Confirm] [] +``` + +### UriPipeline +``` +Copy-AzStorageBlob [-DestBlobType ] -AbsoluteUri -DestContainer -DestBlob + [-StandardBlobTier ] [-RehydratePriority ] [-EncryptionScope ] + [-Context ] [-DestContext ] [-Force] [-AsJob] + [-DefaultProfile ] [-WhatIf] [-Confirm] [] +``` + +## DESCRIPTION +The **Copy-AzStorageBlob** cmdlet copies a blob synchronously, currently only support block blob. + +## EXAMPLES + +### Example 1: Copy a named blob to another +``` +C:\PS> $destBlob = Copy-AzStorageBlob -SrcContainer "sourcecontainername" -SrcBlob "srcblobname" -DestContainer "destcontainername" -DestBlob "destblobname" +``` + +This command copies a blob from source container to the destination container with a new blob name. + +### Example 2: Copy blob from a blob object +``` +C:\PS> $srcBlob = Get-AzStorageBlob -Container $containerName -Blob $blobName -Context $ctx +C:\PS> $destBlob = $srcBlob | Copy-AzStorageBlob -DestContainer "destcontainername" -DestBlob "destblobname" +``` + +This command copies a blob from source blob object to the destination container with a new blob name. + +### Example 3: Copy blob from a blob Uri +``` +C:\PS> $srcBlobUri = New-AzStorageBlobSASToken -Container $srcContainerName -Blob $srcBlobName -Permission rt -ExpiryTime (Get-Date).AddDays(7) -FullUri +C:\PS> $destBlob = Copy-AzStorageBlob -AbsoluteUri $srcBlobUri -DestContainer "destcontainername" -DestBlob "destblobname" +``` + +The first command creates a blob Uri of the source blob, with sas token of permission "rt". The second command copies from source blob Uri to the destination blob. + +### Example 4: Update a block blob encryption scope +``` +C:\PS> $blob = Copy-AzStorageBlob -SrcContainer $containerName -SrcBlob $blobname -DestContainer $containername -EncryptionScope $newScopeName -Force +``` + +This command update a block blob encryption scope by copy it to itself with a new encryption scope. + +## PARAMETERS + +### -AbsoluteUri +Source blob uri + +```yaml +Type: System.String +Parameter Sets: UriPipeline +Aliases: SrcUri, SourceUri + +Required: True +Position: Named +Default value: None +Accept pipeline input: True (ByPropertyName) +Accept wildcard characters: False +``` + +### -AsJob +Run cmdlet in the background + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -BlobBaseClient +BlobBaseClient Object + +```yaml +Type: Azure.Storage.Blobs.Specialized.BlobBaseClient +Parameter Sets: BlobInstance +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: True (ByPropertyName) +Accept wildcard characters: False +``` + +### -Context +Source Azure Storage Context Object + +```yaml +Type: Microsoft.Azure.Commands.Common.Authentication.Abstractions.IStorageContext +Parameter Sets: ContainerName, BlobInstance +Aliases: SrcContext, SourceContext + +Required: False +Position: Named +Default value: None +Accept pipeline input: True (ByPropertyName, ByValue) +Accept wildcard characters: False +``` + +```yaml +Type: Microsoft.Azure.Commands.Common.Authentication.Abstractions.IStorageContext +Parameter Sets: UriPipeline +Aliases: SrcContext, SourceContext + +Required: False +Position: Named +Default value: None +Accept pipeline input: True (ByPropertyName, ByValue) +Accept wildcard characters: False +``` + +### -DefaultProfile +The credentials, account, tenant, and subscription used for communication with Azure. + +```yaml +Type: Microsoft.Azure.Commands.Common.Authentication.Abstractions.Core.IAzureContextContainer +Parameter Sets: (All) +Aliases: AzureRmContext, AzureCredential + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -DestBlob +Destination blob name + +```yaml +Type: System.String +Parameter Sets: ContainerName, BlobInstance +Aliases: DestinationBlob + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +```yaml +Type: System.String +Parameter Sets: UriPipeline +Aliases: DestinationBlob + +Required: True +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -DestBlobType +Destiantion Blob Type('Block', 'Page', 'Append'), by default will be same as source. + +```yaml +Type: System.String +Parameter Sets: (All) +Aliases: +Accepted values: Block, Page, Append + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -DestContainer +Destination container name + +```yaml +Type: System.String +Parameter Sets: (All) +Aliases: DestinationContainer + +Required: True +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -DestContext +Destination Storage context object + +```yaml +Type: Microsoft.Azure.Commands.Common.Authentication.Abstractions.IStorageContext +Parameter Sets: (All) +Aliases: DestinationContext + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -EncryptionScope +Encryption scope to be used when making requests to the dest blob. + +```yaml +Type: System.String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Force +Force to overwrite the existing blob or file + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -RehydratePriority +Block Blob RehydratePriority. +Indicates the priority with which to rehydrate an archived blob. +Valid values are High/Standard. + +```yaml +Type: Microsoft.Azure.Storage.Blob.RehydratePriority +Parameter Sets: (All) +Aliases: +Accepted values: Standard, High + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -SrcBlob +Blob name + +```yaml +Type: System.String +Parameter Sets: ContainerName +Aliases: SourceBlob + +Required: True +Position: 0 +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -SrcContainer +Source Container name + +```yaml +Type: System.String +Parameter Sets: ContainerName +Aliases: SourceContainer + +Required: True +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -StandardBlobTier +Block Blob Tier, valid values are Hot/Cool/Archive. +See detail in https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blob-storage-tiers + +```yaml +Type: System.String +Parameter Sets: (All) +Aliases: +Accepted values: Hot, Cool, Archive + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Confirm +Prompts you for confirmation before running the cmdlet. + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: cf + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -WhatIf +Shows what would happen if the cmdlet runs. +The cmdlet is not run. + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: wi + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### CommonParameters +This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see about_CommonParameters (http://go.microsoft.com/fwlink/?LinkID=113216). + +## INPUTS + +### Azure.Storage.Blobs.Specialized.BlobBaseClient + +### System.String + +### Microsoft.Azure.Commands.Common.Authentication.Abstractions.IStorageContext + +## OUTPUTS + +### Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel.AzureStorageBlob + +## NOTES + +## RELATED LINKS diff --git a/src/Storage/Storage/Blob/Cmdlet/CopyAzureStorageBlob.cs b/src/Storage/Storage/Blob/Cmdlet/CopyAzureStorageBlob.cs new file mode 100644 index 000000000000..d37c2d153d51 --- /dev/null +++ b/src/Storage/Storage/Blob/Cmdlet/CopyAzureStorageBlob.cs @@ -0,0 +1,438 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + + +namespace Microsoft.WindowsAzure.Commands.Storage.Blob.Cmdlet +{ + using Azure.Commands.Common.Authentication.Abstractions; + using Commands.Common.Storage.ResourceModel; + using global::Azure.Storage.Blobs; + using global::Azure.Storage.Blobs.Specialized; + using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters; + using Microsoft.Azure.Storage.Blob; + using Microsoft.WindowsAzure.Commands.Storage.Common; + using Microsoft.WindowsAzure.Commands.Storage.Model.Contract; + using System; + using System.Management.Automation; + using System.Security.Permissions; + using System.Threading.Tasks; + using Track2Models = global::Azure.Storage.Blobs.Models; + + [Cmdlet("Copy", Azure.Commands.ResourceManager.Common.AzureRMConstants.AzurePrefix + "StorageBlob", SupportsShouldProcess = true, DefaultParameterSetName = ContainerNameParameterSet),OutputType(typeof(AzureStorageBlob))] + public class CopyAzureStorageBlob : StorageDataMovementCmdletBase + { + /// + /// Blob Pipeline parameter set name + /// + private const string BlobParameterSet = "BlobInstance"; + + /// + /// Blob name and container name parameter set + /// + private const string ContainerNameParameterSet = "ContainerName"; + + /// + /// Source uri parameter set + /// + private const string UriParameterSet = "UriPipeline"; + + [Parameter(HelpMessage = "BlobBaseClient Object", Mandatory = false, + ValueFromPipelineByPropertyName = true, ParameterSetName = BlobParameterSet)] + [ValidateNotNull] + public BlobBaseClient BlobBaseClient { get; set; } + + [Alias("SourceBlob")] + [Parameter(HelpMessage = "Blob name", ParameterSetName = ContainerNameParameterSet, Mandatory = true, Position = 0)] + public string SrcBlob + { + get { return BlobName; } + set { BlobName = value; } + } + private string BlobName = String.Empty; + + [Alias("SourceContainer")] + [Parameter(HelpMessage = "Source Container name", Mandatory = true, ParameterSetName = ContainerNameParameterSet)] + [ValidateNotNullOrEmpty] + public string SrcContainer + { + get { return ContainerName; } + set { ContainerName = value; } + } + private string ContainerName = String.Empty; + + [Alias("SrcUri", "SourceUri")] + [Parameter(HelpMessage = "Source blob uri", Mandatory = true, + ValueFromPipelineByPropertyName = true, ParameterSetName = UriParameterSet)] + public string AbsoluteUri { get; set; } + + [Alias("DestinationContainer")] + [Parameter(HelpMessage = "Destination container name", Mandatory = true, ParameterSetName = ContainerNameParameterSet)] + [Parameter(HelpMessage = "Destination container name", Mandatory = true, ParameterSetName = UriParameterSet)] + [Parameter(HelpMessage = "Destination container name", Mandatory = true, ParameterSetName = BlobParameterSet)] + //[Parameter(HelpMessage = "Destination container name", Mandatory = true, ParameterSetName = ContainerParameterSet)] + public string DestContainer { get; set; } + + [Alias("DestinationBlob")] + [Parameter(HelpMessage = "Destination blob name", Mandatory = true, ParameterSetName = UriParameterSet)] + [Parameter(HelpMessage = "Destination blob name", Mandatory = false, ParameterSetName = ContainerNameParameterSet)] + [Parameter(HelpMessage = "Destination blob name", Mandatory = false, ParameterSetName = BlobParameterSet)] + //[Parameter(HelpMessage = "Destination blob name", Mandatory = false, ParameterSetName = ContainerParameterSet)] + public string DestBlob { get; set; } + + [Parameter(HelpMessage = "Block Blob Tier, valid values are Hot/Cool/Archive. See detail in https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blob-storage-tiers", Mandatory = false)] + [PSArgumentCompleter("Hot", "Cool", "Archive")] + [ValidateSet("Hot", "Cool", "Archive", IgnoreCase = true)] + public string StandardBlobTier + { + get + { + return standardBlobTier is null ? null : standardBlobTier.Value.ToString(); + } + + set + { + if (value != null) + { + standardBlobTier = ((StandardBlobTier)Enum.Parse(typeof(StandardBlobTier), value, true)); + } + else + { + standardBlobTier = null; + } + } + } + private StandardBlobTier? standardBlobTier = null; + + [Parameter(HelpMessage = "Block Blob RehydratePriority. Indicates the priority with which to rehydrate an archived blob. Valid values are High/Standard.", Mandatory = false)] + [ValidateSet("Standard", "High", IgnoreCase = true)] + public Azure.Storage.Blob.RehydratePriority RehydratePriority + { + get + { + return rehydratePriority.Value; + } + + set + { + rehydratePriority = value; + } + } + private Azure.Storage.Blob.RehydratePriority? rehydratePriority = null; + + [Parameter(HelpMessage = "Encryption scope to be used when making requests to the dest blob.", + Mandatory = false)] + [ValidateNotNullOrEmpty] + public string EncryptionScope { get; set; } + + [Alias("SrcContext", "SourceContext")] + [Parameter(HelpMessage = "Source Azure Storage Context Object", ValueFromPipeline = true, ValueFromPipelineByPropertyName = true, ParameterSetName = ContainerNameParameterSet)] + [Parameter(HelpMessage = "Source Azure Storage Context Object", ValueFromPipeline = true, ValueFromPipelineByPropertyName = true, ParameterSetName = BlobParameterSet)] + [Parameter(HelpMessage = "Source Azure Storage Context Object", ParameterSetName = UriParameterSet)] + public override IStorageContext Context { get; set; } + + [Alias("DestinationContext")] + [Parameter(HelpMessage = "Destination Storage context object", Mandatory = false)] + public IStorageContext DestContext { get; set; } + + public override int? ServerTimeoutPerRequest { get; set; } + public override int? ClientTimeoutPerRequest { get; set; } + public override int? ConcurrentTaskCount { get; set; } + + private bool skipSourceChannelInit; + + /// + /// Create blob client and storage service management channel if need to. + /// + /// IStorageManagement object + protected override IStorageBlobManagement CreateChannel() + { + //Init storage blob management channel + if (skipSourceChannelInit) + { + return null; + } + else + { + return base.CreateChannel(); + } + } + + /// + /// Begin cmdlet processing + /// + protected override void BeginProcessing() + { + if (ParameterSetName == UriParameterSet) + { + skipSourceChannelInit = true; + } + + base.BeginProcessing(); + } + + private IStorageFileManagement GetFileChannel() + { + return new StorageFileManagement(GetCmdletStorageContext()); + } + + /// + /// Set up the Channel object for Destination container and blob + /// + internal IStorageBlobManagement GetDestinationChannel() + { + //If destChannel exits, reuse it. + //If desContext exits, use it. + //If Channl object exists, use it. + //Otherwise, create a new channel. + IStorageBlobManagement destChannel = default(IStorageBlobManagement); + + if (destChannel == null) + { + if (DestContext == null) + { + if (Channel != null) + { + destChannel = Channel; + } + else + { + destChannel = base.CreateChannel(); + } + } + else + { + destChannel = CreateChannel(this.GetCmdletStorageContext(DestContext)); + } + } + + return destChannel; + } + + /// + /// Execute command + /// + [PermissionSet(SecurityAction.Demand, Name = "FullTrust")] + public override void ExecuteCmdlet() + { + IStorageBlobManagement destChannel = GetDestinationChannel(); + IStorageBlobManagement srcChannel = Channel; + + if (!string.IsNullOrEmpty(this.EncryptionScope)) + { + SetClientOptionsWithEncryptionScope(this.EncryptionScope); + } + + string target = string.Empty; + Action copyAction = null; + switch (ParameterSetName) + { + case ContainerNameParameterSet: + copyAction = () => CopyBlobSync(srcChannel, destChannel, SrcContainer, SrcBlob, DestContainer, DestBlob); + target = SrcBlob; + break; + + case UriParameterSet: + copyAction = () => CopyBlobSync(destChannel, AbsoluteUri, DestContainer, DestBlob, (Context != null ? GetCmdletStorageContext(Context) : null)); + target = AbsoluteUri; + break; + + case BlobParameterSet: + copyAction = () => CopyBlobSync(destChannel, BlobBaseClient, DestContainer, DestBlob); + target = BlobBaseClient.Name; + break; + } + + if (copyAction != null && ShouldProcess(target, VerbsCommon.Copy)) + { + copyAction(); + } + } + + private void CopyBlobSync(IStorageBlobManagement SrcChannel, IStorageBlobManagement destChannel, string srcContainerName, string srcBlobName, string destContainerName, string destBlobName) + { + NameUtil.ValidateBlobName(srcBlobName); + NameUtil.ValidateContainerName(srcContainerName); + + BlobBaseClient srcBlobBaseClient = Util.GetTrack2BlobServiceClient(SrcChannel.StorageContext, ClientOptions).GetBlobContainerClient(srcContainerName).GetBlobBaseClient(srcBlobName); + + this.CopyBlobSync(destChannel, srcBlobBaseClient, destContainerName, destBlobName); + } + + private void CopyBlobSync(IStorageBlobManagement destChannel, BlobBaseClient srcBlobBaseClient, string destContainerName, string destBlobName) + { + NameUtil.ValidateContainerName(destContainerName); + if (string.IsNullOrEmpty(destBlobName)) + { + destBlobName = srcBlobBaseClient.Name; + } + else + { + NameUtil.ValidateBlobName(destBlobName); + } + BlobBaseClient destBlob = this.GetDestBlob(destChannel, destContainerName, destBlobName, Util.GetBlobType(srcBlobBaseClient)); + + this.CopyBlobSync(destChannel, srcBlobBaseClient, destBlob); + } + + private void CopyBlobSync(IStorageBlobManagement destChannel, BlobBaseClient srcCloudBlob, BlobBaseClient destCloudBlob) + { + Track2Models.BlobType srcBlobType = Util.GetBlobType(srcCloudBlob, true).Value; + if (srcBlobType != Track2Models.BlobType.Block) + { + throw new ArgumentException(string.Format("The cmdlet currently only support souce blob and destination blob are both block blob. The source blob type is {0}.", srcBlobType)); + } + + if (srcCloudBlob is BlobClient) + { + srcCloudBlob = Util.GetTrack2BlobClientWithType(srcCloudBlob, Channel.StorageContext, srcBlobType, ClientOptions); + } + if (destCloudBlob is BlobClient) + { + destCloudBlob = Util.GetTrack2BlobClientWithType(destCloudBlob, destChannel.StorageContext, srcBlobType, ClientOptions); + } + + Func taskGenerator = (taskId) => CopyFromUri(taskId, destChannel, srcCloudBlob.GenerateUriWithCredentials(Channel.StorageContext), destCloudBlob); + RunTask(taskGenerator); + } + + private void CopyBlobSync(IStorageBlobManagement destChannel, string srcUri, string destContainer, string destBlobName, AzureStorageContext context) + { + Track2Models.BlobType srcBlobType = Util.GetBlobType(new BlobBaseClient(new Uri(srcUri), ClientOptions), true).Value; + if (srcBlobType != Track2Models.BlobType.Block) + { + throw new ArgumentException(string.Format("The cmdlet currently only support souce blob and destination blob are both block blob. The source blob type is {0}.", srcBlobType)); + } + + BlobBaseClient destBlob = this.GetDestBlob(destChannel, destContainer, destBlobName, srcBlobType); + Func taskGenerator = (taskId) => CopyFromUri(taskId, destChannel, new Uri(srcUri), destBlob); + RunTask(taskGenerator); + } + + private async Task CopyFromUri(long taskId, IStorageBlobManagement destChannel, Uri srcUri, BlobBaseClient destBlob) + { + bool destExist = true; + Track2Models.BlobType? destBlobType = Util.GetBlobType(destBlob); + Track2Models.BlobProperties properties = null; + + try + { + properties = (await destBlob.GetPropertiesAsync(this.BlobRequestConditions, cancellationToken: this.CmdletCancellationToken).ConfigureAwait(false)).Value; + destBlobType = properties.BlobType; + } + catch (global::Azure.RequestFailedException e) when (e.Status == 404) + { + destExist = false; + } + if (destBlobType != null) + { + ValidateBlobTier(Util.convertBlobType_Track2ToTrack1(destBlobType), null, standardBlobTier, rehydratePriority); + } + + if (!destExist || this.ConfirmOverwrite(srcUri.AbsoluteUri.ToString(), destBlob.Uri.ToString())) + { + Track2Models.BlobCopyFromUriOptions options = new Track2Models.BlobCopyFromUriOptions(); + + // The Blob Type and Blob Tier must match, since already checked before + if (standardBlobTier != null || rehydratePriority != null) + { + options.AccessTier = Util.ConvertAccessTier_Track1ToTrack2(standardBlobTier); + options.RehydratePriority = Util.ConvertRehydratePriority_Track1ToTrack2(rehydratePriority); + } + options.SourceConditions = this.BlobRequestConditions; + + BlockBlobClient srcBlockblob = new BlockBlobClient(srcUri, ClientOptions); + Track2Models.BlobProperties srcProperties = srcBlockblob.GetProperties(cancellationToken: this.CmdletCancellationToken).Value; + + //Prepare progress handler + string activity = String.Format("Copy Blob {0} to {1}", srcBlockblob.Name, destBlob.Name); + string status = "Prepare to Copy Blob"; + ProgressRecord pr = new ProgressRecord(OutputStream.GetProgressId(taskId), activity, status); + IProgress progressHandler = new Progress((finishedBytes) => + { + if (pr != null) + { + // Size of the source file might be 0, when it is, directly treat the progress as 100 percent. + pr.PercentComplete = 0 == srcProperties.ContentLength ? 100 : (int)(finishedBytes * 100 / srcProperties.ContentLength); + pr.StatusDescription = string.Format("Percent: {0}%.", pr.PercentComplete); + Console.WriteLine(finishedBytes); + this.OutputStream.WriteProgress(pr); + } + }); + + switch (destBlobType) + { + case Track2Models.BlobType.Block: + + BlockBlobClient destBlockBlob = (BlockBlobClient)Util.GetTrack2BlobClientWithType(destBlob, Channel.StorageContext, Track2Models.BlobType.Block, ClientOptions); + + Track2Models.CommitBlockListOptions commitBlockListOptions = new Track2Models.CommitBlockListOptions(); + commitBlockListOptions.HttpHeaders = new Track2Models.BlobHttpHeaders(); + commitBlockListOptions.HttpHeaders.ContentType = srcProperties.ContentType; + commitBlockListOptions.HttpHeaders.ContentHash = srcProperties.ContentHash; + commitBlockListOptions.HttpHeaders.ContentEncoding = srcProperties.ContentEncoding; + commitBlockListOptions.HttpHeaders.ContentLanguage = srcProperties.ContentLanguage; + commitBlockListOptions.HttpHeaders.ContentDisposition = srcProperties.ContentDisposition; + commitBlockListOptions.Metadata = srcProperties.Metadata; + try + { + commitBlockListOptions.Tags = srcBlockblob.GetTags(cancellationToken: this.CmdletCancellationToken).Value.Tags; + } + catch (global::Azure.RequestFailedException e) when (e.Status == 403 || e.Status == 404 || e.Status == 401) + { + if (!this.Force && !OutputStream.ConfirmAsync("Can't get source blob Tags, so source blob tags won't be copied to dest blob. Do you want to continue the blob copy?").Result) + { + return; + } + } + + long blockLength = GetBlockLength(srcProperties.ContentLength); + string[] blockIDs = GetBlockIDs(srcProperties.ContentLength, blockLength, destBlockBlob.Name); + long copyoffset = 0; + progressHandler.Report(copyoffset); + foreach (string id in blockIDs) + { + long blocksize = blockLength; + if (copyoffset + blocksize > srcProperties.ContentLength) + { + blocksize = srcProperties.ContentLength - copyoffset; + } + destBlockBlob.StageBlockFromUri(srcUri, id, new global::Azure.HttpRange(copyoffset, blocksize), cancellationToken: this.CmdletCancellationToken); + copyoffset += blocksize; + progressHandler.Report(copyoffset); + + } + destBlockBlob.CommitBlockList(blockIDs, commitBlockListOptions, this.CmdletCancellationToken); + + break; + case Track2Models.BlobType.Page: + case Track2Models.BlobType.Append: + default: + throw new ArgumentException(string.Format("The cmdlet currently only support souce blob and destination blob are both block blob. The dest blob type is {0}.", destBlobType)); + } + + OutputStream.WriteObject(taskId, new AzureStorageBlob(destBlob, destChannel.StorageContext, null, options: ClientOptions)); + } + } + + private BlobBaseClient GetDestBlob(IStorageBlobManagement destChannel, string destContainerName, string destBlobName, global::Azure.Storage.Blobs.Models.BlobType? blobType) + { + NameUtil.ValidateContainerName(destContainerName); + NameUtil.ValidateBlobName(destBlobName); + + BlobContainerClient container = AzureStorageContainer.GetTrack2BlobContainerClient(destChannel.GetContainerReference(destContainerName), destChannel.StorageContext, ClientOptions); + BlobBaseClient destBlob = Util.GetTrack2BlobClient(container, destBlobName, destChannel.StorageContext, null, null, null, ClientOptions, blobType is null ? global::Azure.Storage.Blobs.Models.BlobType.Block : blobType.Value); + return destBlob; + } + } +} diff --git a/src/Storage/Storage/Blob/Cmdlet/SetAzureStorageBlobContent.cs b/src/Storage/Storage/Blob/Cmdlet/SetAzureStorageBlobContent.cs index f571e26527b3..7bae226cff5b 100644 --- a/src/Storage/Storage/Blob/Cmdlet/SetAzureStorageBlobContent.cs +++ b/src/Storage/Storage/Blob/Cmdlet/SetAzureStorageBlobContent.cs @@ -61,21 +61,6 @@ public class SetAzureBlobContentCommand : StorageDataMovementCmdletBase /// private const string ContainerParameterSet = "ContainerPipeline"; - /// - /// block blob type - /// - private const string BlockBlobType = "Block"; - - /// - /// page blob type - /// - private const string PageBlobType = "Page"; - - /// - /// append blob type - /// - private const string AppendBlobType = "Append"; - [Alias("FullName")] [Parameter(Position = 0, Mandatory = true, HelpMessage = "file Path.", ValueFromPipelineByPropertyName = true, ParameterSetName = ManualParameterSet)] @@ -433,10 +418,7 @@ internal virtual async Task UploadBlobwithSdk(long taskId, IStorageBlobManagemen BlobClientOptions options = this.ClientOptions; if (!string.IsNullOrEmpty(this.EncryptionScope)) { - options = new BlobClientOptions() - { - EncryptionScope = this.EncryptionScope, - }; + options = SetClientOptionsWithEncryptionScope(this.EncryptionScope); } if (this.Force.IsPresent diff --git a/src/Storage/Storage/Blob/StorageCloudBlobCmdletBase.cs b/src/Storage/Storage/Blob/StorageCloudBlobCmdletBase.cs index 6e9f6189c313..d8da32cde2e8 100644 --- a/src/Storage/Storage/Blob/StorageCloudBlobCmdletBase.cs +++ b/src/Storage/Storage/Blob/StorageCloudBlobCmdletBase.cs @@ -78,10 +78,35 @@ public BlobClientOptions ClientOptions { get { - BlobClientOptions options = new BlobClientOptions(); - options.AddPolicy(new UserAgentPolicy(ApiConstants.UserAgentHeaderValue), HttpPipelinePosition.PerCall); - return options; + if (clientOptions == null) + { + clientOptions = new BlobClientOptions(); + clientOptions.AddPolicy(new UserAgentPolicy(ApiConstants.UserAgentHeaderValue), HttpPipelinePosition.PerCall); + return clientOptions; + } + else + { + return clientOptions; + } + } + } + private BlobClientOptions clientOptions = null; + + public BlobClientOptions SetClientOptionsWithEncryptionScope(string encryptionScope) + { + if (clientOptions == null) + { + clientOptions = new BlobClientOptions(); + clientOptions.AddPolicy(new UserAgentPolicy(ApiConstants.UserAgentHeaderValue), HttpPipelinePosition.PerCall); + clientOptions.EncryptionScope = encryptionScope; + return clientOptions; } + else + { + clientOptions.EncryptionScope = encryptionScope; + return clientOptions; + } + } public global::Azure.Storage.Blobs.Models.BlobRequestConditions BlobRequestConditions diff --git a/src/Storage/Storage/Blob/StorageDataMovementCmdletBase.cs b/src/Storage/Storage/Blob/StorageDataMovementCmdletBase.cs index ff51d3595f83..b46bcfdf0a50 100644 --- a/src/Storage/Storage/Blob/StorageDataMovementCmdletBase.cs +++ b/src/Storage/Storage/Blob/StorageDataMovementCmdletBase.cs @@ -21,11 +21,27 @@ namespace Microsoft.WindowsAzure.Commands.Storage.Blob using System.Management.Automation; using System.Threading.Tasks; using OpContext = Microsoft.Azure.Storage.OperationContext; + using System.Collections.Generic; public class StorageDataMovementCmdletBase : StorageCloudBlobCmdletBase, IDisposable { protected const int size4MB = 4 * 1024 * 1024; + /// + /// block blob type + /// + protected const string BlockBlobType = "Block"; + + /// + /// page blob type + /// + protected const string PageBlobType = "Page"; + + /// + /// append blob type + /// + protected const string AppendBlobType = "Append"; + /// /// Blob Transfer Manager /// @@ -156,5 +172,46 @@ protected virtual void DoEndProcessing() this.TransferManager = null; } } + + /// + /// Get the block size from block blob length + /// + public static long GetBlockLength(long contentLength) + { + long blockLength = contentLength / 50000; + if (blockLength % (8 * 1024 * 1024) != 0) + { + blockLength = (blockLength / (8 * 1024 * 1024) + 1) * (8 * 1024 * 1024); + } + return blockLength; + } + + /// + /// Get the block id arrary from block blob length, block size and blob name + /// + public static string[] GetBlockIDs(long contentLength, long blockLength, string blobname) + { + long blockCount = 0; + if (blockLength != 0) + { + blockCount = contentLength / blockLength; + } + if (blockCount * blockLength != contentLength) + { + blockCount++; + } + List blockIDs = new List(); + for (int i = 0; i < (int)blockCount; i++) + { + string idNo = i.ToString(); + while (idNo.Length < 5) + { + idNo = "0" + idNo; + } + string blockID = Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(blobname + idNo)); + blockIDs.Add(blockID); + } + return blockIDs.ToArray(); + } } } diff --git a/src/Storage/Storage/Common/StorageExtensions.cs b/src/Storage/Storage/Common/StorageExtensions.cs index 18c9077c109c..dd65a82d3169 100644 --- a/src/Storage/Storage/Common/StorageExtensions.cs +++ b/src/Storage/Storage/Common/StorageExtensions.cs @@ -10,6 +10,7 @@ namespace Microsoft.WindowsAzure.Commands.Storage.Common { using global::Azure.Storage; + using global::Azure.Storage.Blobs; using global::Azure.Storage.Blobs.Specialized; using global::Azure.Storage.Sas; using Microsoft.Azure.Storage.Auth; @@ -231,7 +232,7 @@ private static string GetBlobSasToken(CloudBlob blob) private static string GetBlobSasToken(BlobBaseClient blob, AzureStorageContext context) { if (null == context.StorageAccount.Credentials - || context.StorageAccount.Credentials.IsAnonymous) + || (context.StorageAccount.Credentials.IsAnonymous && !context.StorageAccount.Credentials.IsToken)) { return string.Empty; } @@ -253,8 +254,27 @@ private static string GetBlobSasToken(BlobBaseClient blob, AzureStorageContext c { sasBuilder.BlobVersionId = Util.GetVersionIdFromBlobUri(blob.Uri); } - sasBuilder.SetPermissions("r"); - string sasToken = sasBuilder.ToSasQueryParameters(new StorageSharedKeyCredential(context.StorageAccountName, context.StorageAccount.Credentials.ExportBase64EncodedKey())).ToString(); + sasBuilder.SetPermissions("rt"); + + string sasToken = null; + if (context != null && context.StorageAccount.Credentials.IsToken) //oauth + { + global::Azure.Storage.Blobs.Models.UserDelegationKey userDelegationKey = null; + BlobServiceClient oauthService = new BlobServiceClient(context.StorageAccount.BlobEndpoint, context.Track2OauthToken, null); + + Util.ValidateUserDelegationKeyStartEndTime(sasBuilder.StartsOn, sasBuilder.ExpiresOn); + + userDelegationKey = oauthService.GetUserDelegationKey( + startsOn: sasBuilder.StartsOn == DateTimeOffset.MinValue ? DateTimeOffset.UtcNow : sasBuilder.StartsOn.ToUniversalTime(), + expiresOn: sasBuilder.ExpiresOn.ToUniversalTime()); + + sasToken = sasBuilder.ToSasQueryParameters(userDelegationKey, context.StorageAccountName).ToString(); + } + else // sharedkey + { + sasToken = sasBuilder.ToSasQueryParameters(new StorageSharedKeyCredential(context.StorageAccountName, context.StorageAccount.Credentials.ExportBase64EncodedKey())).ToString(); + } + if (sasToken[0] != '?') { sasToken = "?" + sasToken;