diff --git a/examples/virtual-hard-disk/01-VirtualHardDisks.ps1 b/examples/virtual-hard-disk/01-VirtualHardDisks.ps1 new file mode 100644 index 000000000000..e4ab5c1b2a79 --- /dev/null +++ b/examples/virtual-hard-disk/01-VirtualHardDisks.ps1 @@ -0,0 +1,6 @@ +Param( + [string]$resourceGroupName, + [string]$resourceGroupLocation +) + +Write-Host "Skip" diff --git a/examples/virtual-hard-disk/01-VirtualHardDisks.sh b/examples/virtual-hard-disk/01-VirtualHardDisks.sh new file mode 100644 index 000000000000..a192e4316d63 --- /dev/null +++ b/examples/virtual-hard-disk/01-VirtualHardDisks.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -e +printf "\n=== Managing Virtual Hard Disks in Azure Compute ===\n" + +printf "\n1. Creating a new resource group: %s and location: %s.\n" "$groupName" "$location" +azure group create -n "$groupName" --location "$location" + +printf "\n2. Creating a new storage account '%s' in type '%s'.\n" "$storageAccountName" "$storageAccountType" +azure storage account new --resourcegroupname "$groupName" --name "$storageAccountName" --location "$location" --type "$storageAccountType" + +printf "\n3. Uploading a virtual hard disk to: %s.\n" "$storageAccountName" +azure vhd add -o --resourcegroupname "$groupName" --destination https://"$storageAccountName".blob.core.windows.net/test/test.vhd --localfilepath $BASEDIR/test.vhd + +printf "\n4. Downloading a virtual hard disk" +azure vhd save -o --resourcegroupname "$groupName" --sourceuri https://"$storageAccountName".blob.core.windows.net/test/test.vhd --localfilepath ./test_downloaded_by_clu.vhd + +printf "\n5. Validating the downloaded file is the same.\n" +diffResult=`diff ./test_downloaded_by_clu.vhd $BASEDIR/test_uploaded_byps.vhd` +if [ "$diffResult" = "" ]; then + echo "Checked" +else + echo "Different!" 1>&2 + exit 1 +fi + +printf "\n6. Removing resource group: %s.\n" "$groupName" +azure group remove -n "$groupName" -f \ No newline at end of file diff --git a/examples/virtual-hard-disk/test.vhd b/examples/virtual-hard-disk/test.vhd new file mode 100644 index 000000000000..a41170a10c6d Binary files /dev/null and b/examples/virtual-hard-disk/test.vhd differ diff --git a/examples/virtual-hard-disk/test_uploaded_byps.vhd b/examples/virtual-hard-disk/test_uploaded_byps.vhd new file mode 100644 index 000000000000..432067ce0dc3 Binary files /dev/null and b/examples/virtual-hard-disk/test_uploaded_byps.vhd differ diff --git a/src/CLU/CLUCoreCLR.sln b/src/CLU/CLUCoreCLR.sln index e8614dbf7ec3..8ea2857d4fff 100644 --- a/src/CLU/CLUCoreCLR.sln +++ b/src/CLU/CLUCoreCLR.sln @@ -57,6 +57,13 @@ Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Microsoft.Azure.Commands.Co EndProject Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Microsoft.ScenarioTest.CLU", "Microsoft.ScenarioTests.CLU\Microsoft.ScenarioTest.CLU.xproj", "{91422B55-28A5-48DE-BCA0-30C3E30FFB1C}" EndProject +Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "VhdManagement", "VhdManagement\VhdManagement.xproj", "{094A32EA-BABC-4A0C-9B6C-3CF7F6EABEC9}" +EndProject +Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Sync", "Sync\Sync.xproj", "{6EDCB32A-8420-48FC-99CE-94BEA12D2FD2}" + ProjectSection(ProjectDependencies) = postProject + {094A32EA-BABC-4A0C-9B6C-3CF7F6EABEC9} = {094A32EA-BABC-4A0C-9B6C-3CF7F6EABEC9} + EndProjectSection +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -147,6 +154,14 @@ Global {91422B55-28A5-48DE-BCA0-30C3E30FFB1C}.Debug|Any CPU.Build.0 = Debug|Any CPU {91422B55-28A5-48DE-BCA0-30C3E30FFB1C}.Release|Any CPU.ActiveCfg = Release|Any CPU {91422B55-28A5-48DE-BCA0-30C3E30FFB1C}.Release|Any CPU.Build.0 = Release|Any CPU + {094A32EA-BABC-4A0C-9B6C-3CF7F6EABEC9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {094A32EA-BABC-4A0C-9B6C-3CF7F6EABEC9}.Debug|Any CPU.Build.0 = Debug|Any CPU + {094A32EA-BABC-4A0C-9B6C-3CF7F6EABEC9}.Release|Any CPU.ActiveCfg = Release|Any CPU + {094A32EA-BABC-4A0C-9B6C-3CF7F6EABEC9}.Release|Any CPU.Build.0 = Release|Any CPU + {6EDCB32A-8420-48FC-99CE-94BEA12D2FD2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6EDCB32A-8420-48FC-99CE-94BEA12D2FD2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6EDCB32A-8420-48FC-99CE-94BEA12D2FD2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6EDCB32A-8420-48FC-99CE-94BEA12D2FD2}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/CLU/Commands.Common.ScenarioTest/ExampleScriptRunner.cs b/src/CLU/Commands.Common.ScenarioTest/ExampleScriptRunner.cs index 8f59cf7b5aec..a10c0a239973 100644 --- a/src/CLU/Commands.Common.ScenarioTest/ExampleScriptRunner.cs +++ b/src/CLU/Commands.Common.ScenarioTest/ExampleScriptRunner.cs @@ -33,6 +33,7 @@ public class ExampleScriptRunner string _sessionId; Random _generator; string _resourceGroupName; + string _storageAccountName; IClientFactory _clientFactory = new ClientFactory(); TestContext _context; ResourceManagementClient _client; @@ -40,6 +41,9 @@ public class ExampleScriptRunner const string ResourceGroupNameKey = "groupName"; const string locationKey = "location"; const string SessionKey = "CmdletSessionID"; + const string storageAccountTypeKey = "storageAccountType"; + const string storageAccountNameKey = "storageAccountName"; + const string DefaultStorageAccountType = "Standard_GRS"; public ExampleScriptRunner(string sessionId) : this(new Random(), sessionId) { @@ -96,6 +100,7 @@ public string RunScript(string testName) { Trace.Listeners.Add(listener); _resourceGroupName = CreateRandomName(); + _storageAccountName = CreateRandomName() + "sto"; if (File.Exists(deploymentTemplatePath)) { DeployTemplate(deploymentTemplatePath, _resourceGroupName); @@ -104,6 +109,8 @@ public string RunScript(string testName) process.EnvironmentVariables[SessionKey] = _sessionId; process.EnvironmentVariables[ResourceGroupNameKey] = _resourceGroupName; process.EnvironmentVariables[locationKey] = DefaultLocation; + process.EnvironmentVariables[storageAccountTypeKey] = DefaultStorageAccountType; + process.EnvironmentVariables[storageAccountNameKey] = _storageAccountName; foreach (var helper in _context.EnvironmentHelpers) { helper.TrySetupScriptEnvironment(_context, _clientFactory, process.EnvironmentVariables); diff --git a/src/CLU/Commands.Common.ScenarioTest/SampleTest.cs b/src/CLU/Commands.Common.ScenarioTest/SampleTest.cs index 48e62f213bea..c727dc7e97f6 100644 --- a/src/CLU/Commands.Common.ScenarioTest/SampleTest.cs +++ b/src/CLU/Commands.Common.ScenarioTest/SampleTest.cs @@ -31,6 +31,7 @@ public SampleTest(ScenarioTestFixture fixture) { _collectionState = fixture; } + [Fact] public void RunSampleTest() { @@ -38,5 +39,11 @@ public void RunSampleTest() helper.RunScript("01-ResourceGroups"); } + [Fact] + public void RunVirtualHardDiskTest() + { + var helper = _collectionState.GetRunner("virtual-hard-disk"); + helper.RunScript("01-VirtualHardDisks"); + } } } diff --git a/src/CLU/Microsoft.Azure.Commands.Compute/Microsoft.Azure.Commands.Compute.xproj b/src/CLU/Microsoft.Azure.Commands.Compute/Microsoft.Azure.Commands.Compute.xproj index 55ef1c9cbefa..fad32b602ac2 100644 --- a/src/CLU/Microsoft.Azure.Commands.Compute/Microsoft.Azure.Commands.Compute.xproj +++ b/src/CLU/Microsoft.Azure.Commands.Compute/Microsoft.Azure.Commands.Compute.xproj @@ -14,5 +14,8 @@ 2.0 + + True + \ No newline at end of file diff --git a/src/CLU/Microsoft.Azure.Commands.Compute/Models/PSSyncOutputEvents.cs b/src/CLU/Microsoft.Azure.Commands.Compute/Models/PSSyncOutputEvents.cs new file mode 100644 index 000000000000..336264bcfd16 --- /dev/null +++ b/src/CLU/Microsoft.Azure.Commands.Compute/Models/PSSyncOutputEvents.cs @@ -0,0 +1,317 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using Microsoft.WindowsAzure.Commands.Sync; +using Microsoft.WindowsAzure.Commands.Sync.Upload; +using Microsoft.WindowsAzure.Commands.Tools.Vhd.Model; +using Microsoft.WindowsAzure.Storage; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Management.Automation; +using System.Management.Automation.Runspaces; +using ProgressRecord = Microsoft.WindowsAzure.Commands.Sync.ProgressRecord; +using Rsrc = Microsoft.Azure.Commands.Compute.Properties.Resources; + +namespace Microsoft.Azure.Commands.Compute.Models +{ + public class PSSyncOutputEvents : ISyncOutputEvents, IDisposable + { + private readonly PSCmdlet cmdlet; + private Runspace runspace; + private bool disposed; + + public PSSyncOutputEvents(PSCmdlet cmdlet) + { + this.cmdlet = cmdlet; + // TODO: CLU + this.runspace = RunspaceFactory.CreateRunspace(InitialSessionState.CreateDefault()); + //this.runspace = RunspaceFactory.CreateRunspace(this.cmdlet.Host); + //this.runspace.Open(); + } + + private static string FormatDuration(TimeSpan ts) + { + if (ts.Days == 0) + { + return String.Format("{0:00}:{1:00}:{2:00}", ts.Hours, ts.Minutes, ts.Seconds); + } + return String.Format(Rsrc.PSSyncOutputEventsFormatDuration, ts.Days, ts.Hours, ts.Minutes, ts.Seconds); + } + + public void ProgressCopyStatus(ProgressRecord record) + { + ProgressCopyStatus(record.PercentComplete, record.AvgThroughputMbPerSecond, record.RemainingTime); + } + + public void ProgressCopyStatus(double precentComplete, double avgThroughputMbps, TimeSpan remainingTime) + { + LogProgress(0, Rsrc.PSSyncOutputEventsCopying, precentComplete, remainingTime, avgThroughputMbps); + } + + public void ProgressCopyComplete(TimeSpan elapsed) + { + LogProgressComplete(0, Rsrc.PSSyncOutputEventsCopying); + LogMessage(Rsrc.PSSyncOutputEventsElapsedTimeForCopy, FormatDuration(elapsed)); + } + + public void ProgressUploadStatus(ProgressRecord record) + { + ProgressUploadStatus(record.PercentComplete, record.AvgThroughputMbPerSecond, record.RemainingTime); + } + + public void ProgressUploadStatus(double precentComplete, double avgThroughputMbps, TimeSpan remainingTime) + { + LogProgress(0, Rsrc.PSSyncOutputEventsUploading, precentComplete, remainingTime, avgThroughputMbps); + } + + private void LogProgress(int activityId, string activity, double precentComplete, TimeSpan remainingTime, double avgThroughputMbps) + { + using (var ps = System.Management.Automation.PowerShell.Create()) + { + ps.Runspace = runspace; + var message = String.Format(Rsrc.PSSyncOutputEventsLogProgress, + precentComplete, + FormatDuration(remainingTime), + avgThroughputMbps); + var progressCommand = String.Format(@"Write-Progress -Id {0} -Activity '{1}' -Status '{2}' -SecondsRemaining {3} -PercentComplete {4}", activityId, activity, message, (int)remainingTime.TotalSeconds, (int)precentComplete); + // TODO: CLU + Console.WriteLine(progressCommand); + //ps.AddScript(progressCommand); + //ps.Invoke(); + } + } + + private void LogProgressComplete(int activityId, string activity) + { + using (var ps = System.Management.Automation.PowerShell.Create()) + { + var progressCommand = String.Format(@"Write-Progress -Id {0} -Activity '{1}' -Status '{2}' -Completed", activityId, activity, Rsrc.PSSyncOutputEventsLogProgressCompleteCompleted); + // TODO: CLU + Console.WriteLine(progressCommand); + ps.Runspace = runspace; + //ps.AddScript(progressCommand); + //ps.Invoke(); + } + } + + public void MessageCreatingNewPageBlob(long pageBlobSize) + { + LogMessage(Rsrc.PSSyncOutputEventsCreatingNewPageBlob, pageBlobSize); + } + + private void LogMessage(string format, params object[] parameters) + { + using (var ps = System.Management.Automation.PowerShell.Create()) + { + ps.Runspace = runspace; + var message = String.Format(format, parameters); + var verboseMessage = String.Format("Write-Host '{0}'", message); + // TODO: CLU + Console.WriteLine(verboseMessage); + //ps.AddScript(verboseMessage); + //ps.Invoke(); + } + } + + private void LogError(Exception e) + { + using (var ps = System.Management.Automation.PowerShell.Create()) + { + ps.Runspace = runspace; + // TODO: CLU + Console.Error.WriteLine(new ErrorRecord(e, String.Empty, ErrorCategory.NotSpecified, null)); + //ps.AddCommand("Write-Error"); + //ps.AddParameter("ErrorRecord", new ErrorRecord(e, String.Empty, ErrorCategory.NotSpecified, null)); + //ps.Invoke(); + } + } + + public void MessageResumingUpload() + { + LogMessage(Rsrc.PSSyncOutputEventsResumingUpload); + } + + public void ProgressUploadComplete(TimeSpan elapsed) + { + LogProgressComplete(0, Rsrc.PSSyncOutputEventsUploading); + LogMessage(Rsrc.PSSyncOutputEventsElapsedTimeForUpload, FormatDuration(elapsed)); + } + + public void ProgressDownloadStatus(ProgressRecord record) + { + ProgressDownloadStatus(record.PercentComplete, record.AvgThroughputMbPerSecond, record.RemainingTime); + } + + public void ProgressDownloadStatus(double precentComplete, double avgThroughputMbps, TimeSpan remainingTime) + { + LogProgress(0, Rsrc.PSSyncOutputEventsDownloading, precentComplete, remainingTime, avgThroughputMbps); + } + + public void ProgressDownloadComplete(TimeSpan elapsed) + { + LogProgressComplete(0, Rsrc.PSSyncOutputEventsDownloading); + LogMessage(Rsrc.PSSyncOutputEventsElapsedTimeForDownload, FormatDuration(elapsed)); + } + + public void ProgressOperationStatus(ProgressRecord record) + { + ProgressOperationStatus(record.PercentComplete, record.AvgThroughputMbPerSecond, record.RemainingTime); + } + + public void ProgressOperationStatus(double percentComplete, double avgThroughputMbps, TimeSpan remainingTime) + { + LogProgress(1, Rsrc.PSSyncOutputEventsCalculatingMD5Hash, percentComplete, remainingTime, avgThroughputMbps); + } + + public void ProgressOperationComplete(TimeSpan elapsed) + { + LogProgressComplete(1, Rsrc.PSSyncOutputEventsCalculatingMD5Hash); + LogMessage(Rsrc.PSSyncOutputEventsElapsedTimeForOperation, FormatDuration(elapsed)); + } + + + public void ErrorUploadFailedWithExceptions(IList exceptions) + { + LogMessage(Rsrc.PSSyncOutputEventsUploadFailedWithException); + foreach (var exception in exceptions) + { + LogError(exception); + } + } + + public void MessageCalculatingMD5Hash(string filePath) + { + LogMessage(Rsrc.PSSyncOutputEventsCalculatingMD5HashForFile, filePath); + } + + public void MessageMD5HashCalculationFinished() + { + LogMessage(Rsrc.PSSyncOutputEventsMD5HashCalculationFinished); + } + + public void MessageRetryingAfterANetworkDisruption() + { + LogMessage(Rsrc.PSSyncOutputEventsRetryingAfterANetworkDisruption); + } + + public void DebugRetryingAfterException(Exception lastException) + { + LogDebug(lastException.ToString()); + + var storageException = lastException as StorageException; + var message = ExceptionUtil.DumpStorageExceptionErrorDetails(storageException); + if (message != String.Empty) + { + LogDebug(message); + } + } + + public void MessageDetectingActualDataBlocks() + { + LogMessage(Rsrc.PSSyncOutputEventsDetectingActualDataBlocks); + } + + public void MessageDetectingActualDataBlocksCompleted() + { + LogMessage(Rsrc.PSSyncOutputEventsDetectingActualDataBlocksCompleted); + } + + public void MessagePrintBlockRange(IndexRange range) + { + LogMessage(Rsrc.PSSyncOutputEventsPrintBlockRange, range, range.Length); + } + + public void DebugEmptyBlockDetected(IndexRange range) + { + LogDebug(Rsrc.PSSyncOutputEventsEmptyBlockDetected, range.ToString()); + } + + private void LogDebug(string format, params object[] parameters) + { + using (var ps = System.Management.Automation.PowerShell.Create()) + { + ps.Runspace = runspace; + // TODO: CLU + var message = String.Format(format, parameters); + var debugMessage = String.Format("Write-Debug -Message '{0}'", message); + Console.WriteLine(debugMessage); + //ps.AddScript(debugMessage); + //ps.Invoke(); + } + } + + public void ProgressEmptyBlockDetection(int processedRangeCount, int totalRangeCount) + { + using (var ps = System.Management.Automation.PowerShell.Create()) + { + if (processedRangeCount >= totalRangeCount) + { + + var progressCommand1 = String.Format(@"Write-Progress -Id {0} -Activity '{1}' -Status '{2}' -Completed", 2, Rsrc.PSSyncOutputEventsProgressEmptyBlockDetection, Rsrc.PSSyncOutputEventsEmptyBlockDetectionCompleted); + ps.Runspace = runspace; + // TODO: CLU + Console.WriteLine(progressCommand1); + //ps.AddScript(progressCommand1); + //ps.Invoke(); + return; + } + + var progressCommand = String.Format(@"Write-Progress -Id {0} -Activity '{1}' -Status '{2}' -SecondsRemaining {3} -PercentComplete {4}", 2, Rsrc.PSSyncOutputEventsProgressEmptyBlockDetection, Rsrc.PSSyncOutputEventsEmptyBlockDetectionDetecting, -1, ((double)processedRangeCount / totalRangeCount) * 100); + ps.Runspace = runspace; + // TODO: CLU + Console.WriteLine(progressCommand); + //ps.AddScript(progressCommand); + //ps.Invoke(); + } + } + + public void WriteVerboseWithTimestamp(string message, params object[] args) + { + // TODO: CLU + var messageWithTimeStamp = string.Format(CultureInfo.CurrentCulture, "{0:T} - {1}", DateTime.Now, string.Format(message, args)); + var progressCommand = String.Format(@"Write-Verbose -Message {0}", messageWithTimeStamp); + Console.WriteLine(progressCommand); + /* + using (var ps = System.Management.Automation.PowerShell.Create()) + { + ps.Runspace = runspace; + // TODO: CLU + //ps.AddScript(progressCommand); + //ps.Invoke(); + } + */ + } + + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (!disposed) + { + if (disposing) + { + // TODO: CLU + //runspace.Dispose(); + } + this.disposed = true; + } + } + } +} diff --git a/src/CLU/Microsoft.Azure.Commands.Compute/Models/UploadParameters.cs b/src/CLU/Microsoft.Azure.Commands.Compute/Models/UploadParameters.cs new file mode 100644 index 000000000000..720299c428f8 --- /dev/null +++ b/src/CLU/Microsoft.Azure.Commands.Compute/Models/UploadParameters.cs @@ -0,0 +1,46 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using Microsoft.Azure.Commands.Compute.StorageServices; +using Microsoft.WindowsAzure.Commands.Sync.Download; +using System.IO; + +namespace Microsoft.Azure.Commands.Compute.Models +{ + public class UploadParameters + { + public UploadParameters(BlobUri destinationUri, BlobUri baseImageUri, FileInfo localFilePath, bool overWrite, int numberOfUploaderThreads) + { + DestinationUri = destinationUri; + BaseImageUri = baseImageUri; + LocalFilePath = localFilePath; + OverWrite = overWrite; + NumberOfUploaderThreads = numberOfUploaderThreads; + } + + public BlobUri DestinationUri { get; private set; } + + public BlobUri BaseImageUri { get; private set; } + + public FileInfo LocalFilePath { get; private set; } + + public bool OverWrite { get; private set; } + + public int NumberOfUploaderThreads { get; private set; } + + public AddAzureVhdCommand Cmdlet { get; set; } + + public CloudPageBlobObjectFactory BlobObjectFactory { get; set; } + } +} diff --git a/src/CLU/Microsoft.Azure.Commands.Compute/Models/VhdDownloadContext.cs b/src/CLU/Microsoft.Azure.Commands.Compute/Models/VhdDownloadContext.cs index 6b460684400e..4fb96535670a 100644 --- a/src/CLU/Microsoft.Azure.Commands.Compute/Models/VhdDownloadContext.cs +++ b/src/CLU/Microsoft.Azure.Commands.Compute/Models/VhdDownloadContext.cs @@ -19,7 +19,7 @@ namespace Microsoft.Azure.Commands.Compute.Models { public class VhdDownloadContext { - public FileInfo LocalFilePath { get; set; } - public Uri Source { get; set; } + public string LocalFilePath { get; set; } + public string Source { get; set; } } } diff --git a/src/CLU/Microsoft.Azure.Commands.Compute/Models/VhdDownloaderModel.cs b/src/CLU/Microsoft.Azure.Commands.Compute/Models/VhdDownloaderModel.cs new file mode 100644 index 000000000000..c2e9272cd35c --- /dev/null +++ b/src/CLU/Microsoft.Azure.Commands.Compute/Models/VhdDownloaderModel.cs @@ -0,0 +1,40 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using Microsoft.WindowsAzure.Commands.Sync; +using Microsoft.WindowsAzure.Commands.Sync.Download; +using System.IO; + +namespace Microsoft.Azure.Commands.Compute.Models +{ + public class VhdDownloaderModel + { + public static VhdDownloadContext Download(DownloaderParameters downloadParameters, ComputeClientBaseCmdlet cmdlet) + { + Program.SyncOutput = new PSSyncOutputEvents(cmdlet); + + downloadParameters.ProgressDownloadComplete = Program.SyncOutput.ProgressDownloadComplete; + downloadParameters.ProgressDownloadStatus = Program.SyncOutput.ProgressDownloadStatus; + + var downloader = new Downloader(downloadParameters); + downloader.Download(); + + return new VhdDownloadContext + { + LocalFilePath = downloadParameters.LocalFilePath, + Source = downloadParameters.BlobUri.Uri.ToString() + }; + } + } +} diff --git a/src/CLU/Microsoft.Azure.Commands.Compute/Models/VhdUploadContext.cs b/src/CLU/Microsoft.Azure.Commands.Compute/Models/VhdUploadContext.cs index bbdd61a3a514..1a77a4ff99f8 100644 --- a/src/CLU/Microsoft.Azure.Commands.Compute/Models/VhdUploadContext.cs +++ b/src/CLU/Microsoft.Azure.Commands.Compute/Models/VhdUploadContext.cs @@ -19,7 +19,7 @@ namespace Microsoft.Azure.Commands.Compute.Models { public class VhdUploadContext { - public FileInfo LocalFilePath { get; set; } - public Uri DestinationUri { get; set; } + public string LocalFilePath { get; set; } + public string DestinationUri { get; set; } } } diff --git a/src/CLU/Microsoft.Azure.Commands.Compute/Models/VhdUploaderModel.cs b/src/CLU/Microsoft.Azure.Commands.Compute/Models/VhdUploaderModel.cs new file mode 100644 index 000000000000..103f21f6dda7 --- /dev/null +++ b/src/CLU/Microsoft.Azure.Commands.Compute/Models/VhdUploaderModel.cs @@ -0,0 +1,47 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using Microsoft.WindowsAzure.Commands.Sync; +using Microsoft.WindowsAzure.Commands.Sync.Upload; + +namespace Microsoft.Azure.Commands.Compute.Models +{ + public class VhdUploaderModel + { + public static VhdUploadContext Upload(UploadParameters uploadParameters) + { + Program.SyncOutput = new PSSyncOutputEvents(uploadParameters.Cmdlet); + + BlobCreatorBase blobCreator; + if (uploadParameters.BaseImageUri != null) + { + blobCreator = new PatchingBlobCreator(uploadParameters.LocalFilePath, uploadParameters.DestinationUri, uploadParameters.BaseImageUri, uploadParameters.BlobObjectFactory, uploadParameters.OverWrite); + } + else + { + blobCreator = new BlobCreator(uploadParameters.LocalFilePath, uploadParameters.DestinationUri, uploadParameters.BlobObjectFactory, uploadParameters.OverWrite); + } + + using (var uploadContext = blobCreator.Create()) + { + var synchronizer = new BlobSynchronizer(uploadContext, uploadParameters.NumberOfUploaderThreads); + if (synchronizer.Synchronize()) + { + return new VhdUploadContext { LocalFilePath = uploadParameters.LocalFilePath.ToString(), DestinationUri = uploadParameters.DestinationUri.Uri.ToString() }; + } + return null; + } + } + } +} diff --git a/src/CLU/Microsoft.Azure.Commands.Compute/Properties/Resources.Designer.cs b/src/CLU/Microsoft.Azure.Commands.Compute/Properties/Resources.Designer.cs index 9cabaca11d5b..fd6999fcd8e6 100644 --- a/src/CLU/Microsoft.Azure.Commands.Compute/Properties/Resources.Designer.cs +++ b/src/CLU/Microsoft.Azure.Commands.Compute/Properties/Resources.Designer.cs @@ -60,11 +60,900 @@ internal Resources() { } /// - /// Looks up a localized string similar to Test. + /// Looks up a localized string similar to SAS Uri for the destination blob is not supported in patch mode:{0}. /// - internal static string Test { + public static string AddAzureVhdCommandSASUriNotSupportedInPatchMode { get { - return ResourceManager.GetString("Test", resourceCulture); + return ResourceManager.GetString("AddAzureVhdCommandSASUriNotSupportedInPatchMode", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Availability set removal operation. + /// + public static string AvailabilitySetRemovalCaption { + get { + return ResourceManager.GetString("AvailabilitySetRemovalCaption", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to This cmdlet will remove the specified availability set. Do you want to continue?. + /// + public static string AvailabilitySetRemovalConfirmation { + get { + return ResourceManager.GetString("AvailabilitySetRemovalConfirmation", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Apply configuration '{0}'. + /// + public static string AzureVMDscApplyConfigurationAction { + get { + return ResourceManager.GetString("AzureVMDscApplyConfigurationAction", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to File '{0}' already exists. Use the -Force parameter to overwrite it.. + /// + public static string AzureVMDscArchiveAlreadyExists { + get { + return ResourceManager.GetString("AzureVMDscArchiveAlreadyExists", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Cannot find configuration data file: {0}. + /// + public static string AzureVMDscCannotFindConfigurationDataFile { + get { + return ResourceManager.GetString("AzureVMDscCannotFindConfigurationDataFile", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to The -ConfigurationArchive parameter must no include a path. + /// + public static string AzureVMDscConfigurationDataFileShouldNotIncludePath { + get { + return ResourceManager.GetString("AzureVMDscConfigurationDataFileShouldNotIncludePath", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Create Archive. + /// + public static string AzureVMDscCreateArchiveAction { + get { + return ResourceManager.GetString("AzureVMDscCreateArchiveAction", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Can not find your azure storage credential. Please specify an storage context using the -StorageContext parameter, or set the current storage account using "Set-AzureRmSubscription", or set the "AZURE_STORAGE_CONNECTION_STRING" environment variable.. + /// + public static string AzureVMDscDefaultStorageCredentialsNotFound { + get { + return ResourceManager.GetString("AzureVMDscDefaultStorageCredentialsNotFound", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Please enter a valid DSC Extension version. Allowed format: N.N where N = [1..9]. + /// + public static string AzureVMDscExtensionInvalidVersion { + get { + return ResourceManager.GetString("AzureVMDscExtensionInvalidVersion", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to The configuration data must be a .psd1 file. + /// + public static string AzureVMDscInvalidConfigurationDataFile { + get { + return ResourceManager.GetString("AzureVMDscInvalidConfigurationDataFile", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to If the ConfigurationArchive argument is null, then the ConfigurationName, ConfigurationArgument, and ConfigurationDataPath parameters must not be specified . + /// + public static string AzureVMDscNullArchiveNoConfiguragionParameters { + get { + return ResourceManager.GetString("AzureVMDscNullArchiveNoConfiguragionParameters", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to If the ConfigurationArchive argument is null, then the StorageContext, ArchiveContainerName, and ArchiveStorageEndpointSuffix parameters must not be specified. + /// + public static string AzureVMDscNullArchiveNoStorageParameters { + get { + return ResourceManager.GetString("AzureVMDscNullArchiveNoStorageParameters", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Parsing configuration script: {0}. + /// + public static string AzureVMDscParsingConfiguration { + get { + return ResourceManager.GetString("AzureVMDscParsingConfiguration", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Storage Blob '{0}' already exists. Use the -Force parameter to overwrite it.. + /// + public static string AzureVMDscStorageBlobAlreadyExists { + get { + return ResourceManager.GetString("AzureVMDscStorageBlobAlreadyExists", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to The storage context must include an storage account.. + /// + public static string AzureVMDscStorageContextMustIncludeAccountName { + get { + return ResourceManager.GetString("AzureVMDscStorageContextMustIncludeAccountName", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Upload '{0}'. + /// + public static string AzureVMDscUploadToBlobStorageAction { + get { + return ResourceManager.GetString("AzureVMDscUploadToBlobStorageAction", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Cannot deserialize settings string from DSC extension. Updating your Azure PowerShell SDK to the latest version may solve this problem. Settings string: + ///{0}. + /// + public static string AzureVMDscWrongSettingsFormat { + get { + return ResourceManager.GetString("AzureVMDscWrongSettingsFormat", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Cannot deserialize settings string from Sql Server extension. Updating your Azure PowerShell SDK to the latest version may solve this problem. Settings string: + ///{0}. + /// + public static string AzureVMSqlServerWrongSettingsFormat { + get { + return ResourceManager.GetString("AzureVMSqlServerWrongSettingsFormat", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Storage account name for boot diagnostics is not given.. + /// + public static string BootDiagnosticsNoStorageAccountError { + get { + return ResourceManager.GetString("BootDiagnosticsNoStorageAccountError", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Cannot specify both Windows and Linux configurations.. + /// + public static string BothWindowsAndLinuxConfigurationsSpecified { + get { + return ResourceManager.GetString("BothWindowsAndLinuxConfigurationsSpecified", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Since the VM is created using premium storage, new standard storage account, {0}, is created for boot diagnostics.. + /// + public static string CreatingStorageAccountForBootDiagnostics { + get { + return ResourceManager.GetString("CreatingStorageAccountForBootDiagnostics", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to No Run File has been assigned, and the Custom Script extension will try to use the first specified File Name as the Run File.. + /// + public static string CustomScriptExtensionTryToUseTheFirstSpecifiedFileAsRunScript { + get { + return ResourceManager.GetString("CustomScriptExtensionTryToUseTheFirstSpecifiedFileAsRunScript", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to A data disk, {0}, is not currently assigned for this VM. Use Add-AzureRmVMDataDisk to add it.. + /// + public static string DataDiskNotAssignedForVM { + get { + return ResourceManager.GetString("DataDiskNotAssignedForVM", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Remove Extension. + /// + public static string DscExtensionRemovalCaption { + get { + return ResourceManager.GetString("DscExtensionRemovalCaption", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to {0}. + /// + public static string DscExtensionRemovalConfirmation { + get { + return ResourceManager.GetString("DscExtensionRemovalConfirmation", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Enable AzureDiskEncryption on the VM. + /// + public static string EnableAzureDiskEncryptionCaption { + get { + return ResourceManager.GetString("EnableAzureDiskEncryptionCaption", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to This cmdlet prepares the VM and enables encryption which may reboot the machine and takes 10-15 minutes to finish. Please save your work on the VM before confirming. Do you want to continue?. + /// + public static string EnableAzureDiskEncryptionConfirmation { + get { + return ResourceManager.GetString("EnableAzureDiskEncryptionConfirmation", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Error occurred when creating storage account for boot diagnostics. Keep creating a VM with disabling boot diagnostics. : {0}. + /// + public static string ErrorDuringCreatingStorageAccountForBootDiagnostics { + get { + return ResourceManager.GetString("ErrorDuringCreatingStorageAccountForBootDiagnostics", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Error occurred when getting storage account, {0}, for boot diagnostics: {1}. + /// + public static string ErrorDuringGettingStorageAccountForBootDiagnostics { + get { + return ResourceManager.GetString("ErrorDuringGettingStorageAccountForBootDiagnostics", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Storage account, {0}, is a premium account. You cannot specify a premium storage account for boot diagnostics. + /// + public static string PremiumStorageAccountForBootDiagnostics { + get { + return ResourceManager.GetString("PremiumStorageAccountForBootDiagnostics", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Calculating MD5 Hash. + /// + public static string PSSyncOutputEventsCalculatingMD5Hash { + get { + return ResourceManager.GetString("PSSyncOutputEventsCalculatingMD5Hash", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to MD5 hash is being calculated for the file '{0}'.. + /// + public static string PSSyncOutputEventsCalculatingMD5HashForFile { + get { + return ResourceManager.GetString("PSSyncOutputEventsCalculatingMD5HashForFile", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Copying. + /// + public static string PSSyncOutputEventsCopying { + get { + return ResourceManager.GetString("PSSyncOutputEventsCopying", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Creating new page blob of size {0}.... + /// + public static string PSSyncOutputEventsCreatingNewPageBlob { + get { + return ResourceManager.GetString("PSSyncOutputEventsCreatingNewPageBlob", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Detecting the empty data blocks in the local file.. + /// + public static string PSSyncOutputEventsDetectingActualDataBlocks { + get { + return ResourceManager.GetString("PSSyncOutputEventsDetectingActualDataBlocks", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Detecting the empty data blocks completed.. + /// + public static string PSSyncOutputEventsDetectingActualDataBlocksCompleted { + get { + return ResourceManager.GetString("PSSyncOutputEventsDetectingActualDataBlocksCompleted", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Downloading. + /// + public static string PSSyncOutputEventsDownloading { + get { + return ResourceManager.GetString("PSSyncOutputEventsDownloading", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Elapsed time for copy: {0}. + /// + public static string PSSyncOutputEventsElapsedTimeForCopy { + get { + return ResourceManager.GetString("PSSyncOutputEventsElapsedTimeForCopy", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Elapsed time for download: {0}. + /// + public static string PSSyncOutputEventsElapsedTimeForDownload { + get { + return ResourceManager.GetString("PSSyncOutputEventsElapsedTimeForDownload", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Elapsed time for the operation: {0}. + /// + public static string PSSyncOutputEventsElapsedTimeForOperation { + get { + return ResourceManager.GetString("PSSyncOutputEventsElapsedTimeForOperation", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Elapsed time for upload: {0}. + /// + public static string PSSyncOutputEventsElapsedTimeForUpload { + get { + return ResourceManager.GetString("PSSyncOutputEventsElapsedTimeForUpload", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Empty block detected: {0}. + /// + public static string PSSyncOutputEventsEmptyBlockDetected { + get { + return ResourceManager.GetString("PSSyncOutputEventsEmptyBlockDetected", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Completed. + /// + public static string PSSyncOutputEventsEmptyBlockDetectionCompleted { + get { + return ResourceManager.GetString("PSSyncOutputEventsEmptyBlockDetectionCompleted", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Detecting empty blocks. + /// + public static string PSSyncOutputEventsEmptyBlockDetectionDetecting { + get { + return ResourceManager.GetString("PSSyncOutputEventsEmptyBlockDetectionDetecting", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to {0} days {1:00}:{2:00}:{3:00}. + /// + public static string PSSyncOutputEventsFormatDuration { + get { + return ResourceManager.GetString("PSSyncOutputEventsFormatDuration", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to {0:0.0}% complete; Remaining Time: {1}; Throughput: {2:0.0}Mbps. + /// + public static string PSSyncOutputEventsLogProgress { + get { + return ResourceManager.GetString("PSSyncOutputEventsLogProgress", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Completed. + /// + public static string PSSyncOutputEventsLogProgressCompleteCompleted { + get { + return ResourceManager.GetString("PSSyncOutputEventsLogProgressCompleteCompleted", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to MD5 hash calculation is completed.. + /// + public static string PSSyncOutputEventsMD5HashCalculationFinished { + get { + return ResourceManager.GetString("PSSyncOutputEventsMD5HashCalculationFinished", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Range of the block is {0}, Length: {1}. + /// + public static string PSSyncOutputEventsPrintBlockRange { + get { + return ResourceManager.GetString("PSSyncOutputEventsPrintBlockRange", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Empty Block Detection. + /// + public static string PSSyncOutputEventsProgressEmptyBlockDetection { + get { + return ResourceManager.GetString("PSSyncOutputEventsProgressEmptyBlockDetection", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Found existing page blob. Resuming upload.... + /// + public static string PSSyncOutputEventsResumingUpload { + get { + return ResourceManager.GetString("PSSyncOutputEventsResumingUpload", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Network disruption occured, retrying.. + /// + public static string PSSyncOutputEventsRetryingAfterANetworkDisruption { + get { + return ResourceManager.GetString("PSSyncOutputEventsRetryingAfterANetworkDisruption", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Upload failed with exceptions:. + /// + public static string PSSyncOutputEventsUploadFailedWithException { + get { + return ResourceManager.GetString("PSSyncOutputEventsUploadFailedWithException", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Uploading. + /// + public static string PSSyncOutputEventsUploading { + get { + return ResourceManager.GetString("PSSyncOutputEventsUploading", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Path '{0}' not found.. + /// + public static string PublishVMDscExtensionAdditionalContentPathNotExist { + get { + return ResourceManager.GetString("PublishVMDscExtensionAdditionalContentPathNotExist", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Configuration published to {0}. + /// + public static string PublishVMDscExtensionArchiveUploadedMessage { + get { + return ResourceManager.GetString("PublishVMDscExtensionArchiveUploadedMessage", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Configuration published to {0}. + /// + public static string PublishVMDscExtensionArchiveUploadedMessage1 { + get { + return ResourceManager.GetString("PublishVMDscExtensionArchiveUploadedMessage1", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Cannot find configuration file: {0}.. + /// + public static string PublishVMDscExtensionConfigFileNotFound { + get { + return ResourceManager.GetString("PublishVMDscExtensionConfigFileNotFound", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Copy '{0}' to '{1}'.. + /// + public static string PublishVMDscExtensionCopyFileVerbose { + get { + return ResourceManager.GetString("PublishVMDscExtensionCopyFileVerbose", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Copy '{0}' to '{1}'.. + /// + public static string PublishVMDscExtensionCopyFileVerbose1 { + get { + return ResourceManager.GetString("PublishVMDscExtensionCopyFileVerbose1", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Copy the module '{0}' to '{1}'.. + /// + public static string PublishVMDscExtensionCopyModuleVerbose { + get { + return ResourceManager.GetString("PublishVMDscExtensionCopyModuleVerbose", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Copy the module '{0}' to '{1}'.. + /// + public static string PublishVMDscExtensionCopyModuleVerbose1 { + get { + return ResourceManager.GetString("PublishVMDscExtensionCopyModuleVerbose1", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Invalid configuration file: {0}. + ///The file needs to be a PowerShell script (.ps1 or .psm1).. + /// + public static string PublishVMDscExtensionCreateArchiveConfigFileInvalidExtension { + get { + return ResourceManager.GetString("PublishVMDscExtensionCreateArchiveConfigFileInvalidExtension", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Invalid configuration file: {0}. + ///The file needs to be a PowerShell script (.ps1 or .psm1).. + /// + public static string PublishVMDscExtensionCreateArchiveConfigFileInvalidExtension1 { + get { + return ResourceManager.GetString("PublishVMDscExtensionCreateArchiveConfigFileInvalidExtension1", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Create a zip file '{0}' from directory '{1}'.. + /// + public static string PublishVMDscExtensionCreateZipVerbose { + get { + return ResourceManager.GetString("PublishVMDscExtensionCreateZipVerbose", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Deleted '{0}'. + /// + public static string PublishVMDscExtensionDeletedFileMessage { + get { + return ResourceManager.GetString("PublishVMDscExtensionDeletedFileMessage", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Deleted '{0}'. + /// + public static string PublishVMDscExtensionDeletedFileMessage1 { + get { + return ResourceManager.GetString("PublishVMDscExtensionDeletedFileMessage1", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Cannot delete '{0}': {1}. + /// + public static string PublishVMDscExtensionDeleteErrorMessage { + get { + return ResourceManager.GetString("PublishVMDscExtensionDeleteErrorMessage", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Cannot delete '{0}': {1}. + /// + public static string PublishVMDscExtensionDeleteErrorMessage1 { + get { + return ResourceManager.GetString("PublishVMDscExtensionDeleteErrorMessage1", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Path '{0}' not found.. + /// + public static string PublishVMDscExtensionDirectoryNotExist { + get { + return ResourceManager.GetString("PublishVMDscExtensionDirectoryNotExist", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Cannot get module for DscResource '{0}'. Possible solutions: + ///1) Specify -ModuleName for Import-DscResource in your configuration. + ///2) Unblock module that contains resource. + ///3) Move Import-DscResource inside Node block. + ///. + /// + public static string PublishVMDscExtensionGetDscResourceFailed { + get { + return ResourceManager.GetString("PublishVMDscExtensionGetDscResourceFailed", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Cannot get module for DscResource '{0}'. Possible solutions: + ///1) Specify -ModuleName for Import-DscResource in your configuration. + ///2) Unblock module that contains resource. + ///3) Move Import-DscResource inside Node block. + ///. + /// + public static string PublishVMDscExtensionGetDscResourceFailed1 { + get { + return ResourceManager.GetString("PublishVMDscExtensionGetDscResourceFailed1", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to List of required modules: [{0}].. + /// + public static string PublishVMDscExtensionRequiredModulesVerbose { + get { + return ResourceManager.GetString("PublishVMDscExtensionRequiredModulesVerbose", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to List of required modules: [{0}].. + /// + public static string PublishVMDscExtensionRequiredModulesVerbose1 { + get { + return ResourceManager.GetString("PublishVMDscExtensionRequiredModulesVerbose1", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Your current PowerShell version {1} is less then required by this cmdlet {0}. Consider download and install latest PowerShell version.. + /// + public static string PublishVMDscExtensionRequiredPsVersion { + get { + return ResourceManager.GetString("PublishVMDscExtensionRequiredPsVersion", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Your current PowerShell version {1} is less then required by this cmdlet {0}. Consider download and install latest PowerShell version.. + /// + public static string PublishVMDscExtensionRequiredPsVersion1 { + get { + return ResourceManager.GetString("PublishVMDscExtensionRequiredPsVersion1", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Configuration script '{0}' contained parse errors: + ///{1}. + /// + public static string PublishVMDscExtensionStorageParserErrors { + get { + return ResourceManager.GetString("PublishVMDscExtensionStorageParserErrors", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Configuration script '{0}' contained parse errors: + ///{1}. + /// + public static string PublishVMDscExtensionStorageParserErrors1 { + get { + return ResourceManager.GetString("PublishVMDscExtensionStorageParserErrors1", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Temp folder '{0}' created.. + /// + public static string PublishVMDscExtensionTempFolderVerbose { + get { + return ResourceManager.GetString("PublishVMDscExtensionTempFolderVerbose", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Temp folder '{0}' created.. + /// + public static string PublishVMDscExtensionTempFolderVerbose1 { + get { + return ResourceManager.GetString("PublishVMDscExtensionTempFolderVerbose1", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Invalid configuration file: {0}. + ///The file needs to be a PowerShell script (.ps1 or .psm1) or a ZIP archive (.zip).. + /// + public static string PublishVMDscExtensionUploadArchiveConfigFileInvalidExtension { + get { + return ResourceManager.GetString("PublishVMDscExtensionUploadArchiveConfigFileInvalidExtension", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Invalid configuration file: {0}. + ///The file needs to be a PowerShell script (.ps1 or .psm1) or a ZIP archive (.zip).. + /// + public static string PublishVMDscExtensionUploadArchiveConfigFileInvalidExtension1 { + get { + return ResourceManager.GetString("PublishVMDscExtensionUploadArchiveConfigFileInvalidExtension1", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Configuration file '{0}' not found.. + /// + public static string PublishVMDscExtensionUploadArchiveConfigFileNotExist { + get { + return ResourceManager.GetString("PublishVMDscExtensionUploadArchiveConfigFileNotExist", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Configuration file '{0}' not found.. + /// + public static string PublishVMDscExtensionUploadArchiveConfigFileNotExist1 { + get { + return ResourceManager.GetString("PublishVMDscExtensionUploadArchiveConfigFileNotExist1", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Storage account, {0}, is not found. The OS disk may be in a different storage group.. + /// + public static string StorageAccountNotFoundForBootDiagnostics { + get { + return ResourceManager.GetString("StorageAccountNotFoundForBootDiagnostics", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to No current subscription has been designated. Use Select-AzureRmSubscription -Current <subscriptionName> to set the current subscription.. + /// + public static string StorageCredentialsFactoryCurrentSubscriptionNotSet { + get { + return ResourceManager.GetString("StorageCredentialsFactoryCurrentSubscriptionNotSet", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Since the VM is created using premium storage, existing standard storage account, {0}, is used for boot diagnostics.. + /// + public static string UsingExistingStorageAccountForBootDiagnostics { + get { + return ResourceManager.GetString("UsingExistingStorageAccountForBootDiagnostics", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Virtual machine extension removal operation. + /// + public static string VirtualMachineExtensionRemovalCaption { + get { + return ResourceManager.GetString("VirtualMachineExtensionRemovalCaption", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to This cmdlet will remove the specified virtual machine extension. Do you want to continue?. + /// + public static string VirtualMachineExtensionRemovalConfirmation { + get { + return ResourceManager.GetString("VirtualMachineExtensionRemovalConfirmation", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to The RDP file cannot be generated because the network interface of the virtual machine does not reference a PublicIP or an InboundNatRule of a public load balancer. . + /// + public static string VirtualMachineNotAssociatedWithPublicIPOrPublicLoadBalancer { + get { + return ResourceManager.GetString("VirtualMachineNotAssociatedWithPublicIPOrPublicLoadBalancer", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to The RDP file cannot be generated because the network interface of the virtual machine does not reference an InboundNatRule of a public load balancer.. + /// + public static string VirtualMachineNotAssociatedWithPublicLoadBalancer { + get { + return ResourceManager.GetString("VirtualMachineNotAssociatedWithPublicLoadBalancer", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to The RDP file cannot be generated because the network interface of the virtual machine does not reference a PublicIP or an InboungNatRule of the load balancer.. + /// + public static string VirtualMachineReferencesInternalNetworkInterface { + get { + return ResourceManager.GetString("VirtualMachineReferencesInternalNetworkInterface", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Virtual machine removal operation. + /// + public static string VirtualMachineRemovalCaption { + get { + return ResourceManager.GetString("VirtualMachineRemovalCaption", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to This cmdlet will remove the specified virtual machine. Do you want to continue?. + /// + public static string VirtualMachineRemovalConfirmation { + get { + return ResourceManager.GetString("VirtualMachineRemovalConfirmation", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Virtual machine stopping operation. + /// + public static string VirtualMachineStoppingCaption { + get { + return ResourceManager.GetString("VirtualMachineStoppingCaption", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to This cmdlet will stop the specified virtual machine. Do you want to continue?. + /// + public static string VirtualMachineStoppingConfirmation { + get { + return ResourceManager.GetString("VirtualMachineStoppingConfirmation", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to You have to specify either both of KeyEncryptionKeyVaultId and KeyEncryptionKeyUrl or none of them.. + /// + public static string VMOSDiskDiskEncryptionBothKekVaultIdAndKekUrlRequired { + get { + return ResourceManager.GetString("VMOSDiskDiskEncryptionBothKekVaultIdAndKekUrlRequired", resourceCulture); } } } diff --git a/src/CLU/Microsoft.Azure.Commands.Compute/StorageServices/AddAzureVhdCommand.cs b/src/CLU/Microsoft.Azure.Commands.Compute/StorageServices/AddAzureVhdCommand.cs index 4411cecccca3..d464623e5aeb 100644 --- a/src/CLU/Microsoft.Azure.Commands.Compute/StorageServices/AddAzureVhdCommand.cs +++ b/src/CLU/Microsoft.Azure.Commands.Compute/StorageServices/AddAzureVhdCommand.cs @@ -12,166 +12,171 @@ // limitations under the License. // ---------------------------------------------------------------------------------- -// TODO: Rewrite for removing Sync -//using Microsoft.Azure.Commands.Compute.Common; -//using Microsoft.Azure.Commands.Compute.Models; -//using Microsoft.Azure.Common.Authentication; -//using Microsoft.Azure.Common.Authentication.Models; -//using Microsoft.WindowsAzure.Commands.Sync.Download; -//using System; -//using System.IO; -//using System.Management.Automation; -//using Rsrc = Microsoft.Azure.Commands.Compute.Properties.Resources; -//using Microsoft.Azure.Management.Storage; -//using Microsoft.Azure.Commands.ResourceManager.Common; - -//namespace Microsoft.Azure.Commands.Compute.StorageServices -//{ -// /// -// /// Uploads a vhd as fixed disk format vhd to a blob in Microsoft Azure Storage -// /// -// [Cmdlet(VerbsCommon.Add, ProfileNouns.Vhd), OutputType(typeof(VhdUploadContext))] -// public class AddAzureVhdCommand : ComputeClientBaseCmdlet -// { -// private const int DefaultNumberOfUploaderThreads = 8; - -// [Parameter( -// Position = 0, -// Mandatory = false, -// ValueFromPipelineByPropertyName = true)] -// [ValidateNotNullOrEmpty] -// public string ResourceGroupName { get; set; } - -// [Parameter( -// Position = 1, -// Mandatory = true, -// ValueFromPipelineByPropertyName = true, -// HelpMessage = "Uri to blob")] -// [ValidateNotNullOrEmpty] -// [Alias("dst")] -// public Uri Destination -// { -// get; -// set; -// } - -// [Parameter( -// Position = 2, -// Mandatory = true, -// ValueFromPipelineByPropertyName = true, -// HelpMessage = "Local path of the vhd file")] -// [ValidateNotNullOrEmpty] -// [Alias("lf")] -// public FileInfo LocalFilePath -// { -// get; -// set; -// } - -// [Parameter( -// Position = 3, -// Mandatory = false, -// ValueFromPipelineByPropertyName = true, -// HelpMessage = "Number of uploader threads")] -// [ValidateNotNullOrEmpty] -// [ValidateRange(1, 64)] -// [Alias("th")] -// public int? NumberOfUploaderThreads -// { -// get; -// set; -// } - -// [Parameter( -// Position = 4, -// Mandatory = false, -// ValueFromPipelineByPropertyName = true, -// HelpMessage = "Uri to a base image in a blob storage account to apply the difference")] -// [ValidateNotNullOrEmpty] -// [Alias("bs")] -// public Uri BaseImageUriToPatch -// { -// get; -// set; -// } - -// [Parameter( -// Position = 5, -// Mandatory = false, -// ValueFromPipelineByPropertyName = true, -// ParameterSetName="Vhd", -// HelpMessage = "Delete the blob if already exists")] -// [ValidateNotNullOrEmpty] -// [Alias("o")] -// public SwitchParameter OverWrite -// { -// get; -// set; -// } - -// public UploadParameters ValidateParameters() -// { -// BlobUri destinationUri; -// if (!BlobUri.TryParseUri(Destination, out destinationUri)) -// { -// throw new ArgumentOutOfRangeException("Destination", this.Destination.ToString()); -// } - -// BlobUri baseImageUri = null; -// if (this.BaseImageUriToPatch != null) -// { -// if (!BlobUri.TryParseUri(BaseImageUriToPatch, out baseImageUri)) -// { -// throw new ArgumentOutOfRangeException("BaseImageUriToPatch", this.BaseImageUriToPatch.ToString()); -// } - -// if (!String.IsNullOrEmpty(destinationUri.Uri.Query)) -// { -// var message = String.Format(Rsrc.AddAzureVhdCommandSASUriNotSupportedInPatchMode, destinationUri.Uri); -// throw new ArgumentOutOfRangeException("Destination", message); -// } -// } - -// var storageCredentialsFactory = CreateStorageCredentialsFactory(); - -// PathIntrinsics currentPath = SessionState.Path; -// var filePath = new FileInfo(currentPath.GetUnresolvedProviderPathFromPSPath(LocalFilePath.ToString())); - -// var parameters = new UploadParameters( -// destinationUri, baseImageUri, filePath, OverWrite.IsPresent, -// (NumberOfUploaderThreads) ?? DefaultNumberOfUploaderThreads) -// { -// Cmdlet = this, -// BlobObjectFactory = new CloudPageBlobObjectFactory(storageCredentialsFactory, TimeSpan.FromMinutes(1)) -// }; - -// return parameters; -// } - -// private StorageCredentialsFactory CreateStorageCredentialsFactory() -// { -// StorageCredentialsFactory storageCredentialsFactory; - -// var storageClient = AzureSession.ClientFactory.CreateClient( -// DefaultProfile.Context, AzureEnvironment.Endpoint.ResourceManager); - -// if (StorageCredentialsFactory.IsChannelRequired(Destination)) -// { -// storageCredentialsFactory = new StorageCredentialsFactory(this.ResourceGroupName, storageClient, DefaultContext.Subscription); -// } -// else -// { -// storageCredentialsFactory = new StorageCredentialsFactory(); -// } - -// return storageCredentialsFactory; -// } - -// protected override void ProcessRecord() -// { -// var parameters = ValidateParameters(); -// var vhdUploadContext = VhdUploaderModel.Upload(parameters); -// WriteObject(vhdUploadContext); -// } -// } -//} +using Microsoft.Azure.Commands.Compute.Common; +using Microsoft.Azure.Commands.Compute.Models; +using Microsoft.WindowsAzure.Commands.Sync.Download; +using System; +using System.IO; +using System.Management.Automation; +using Rsrc = Microsoft.Azure.Commands.Compute.Properties.Resources; +using Microsoft.Azure.Management.Storage; +using Microsoft.Azure.Commands.Common.Authentication.Models; + +namespace Microsoft.Azure.Commands.Compute.StorageServices +{ + /// + /// Uploads a vhd as fixed disk format vhd to a blob in Microsoft Azure Storage + /// + [Cmdlet(VerbsCommon.Add, ProfileNouns.Vhd), OutputType(typeof(VhdUploadContext))] + public class AddAzureVhdCommand : ComputeClientBaseCmdlet + { + private const int DefaultNumberOfUploaderThreads = 8; + + [Parameter( + Position = 0, + Mandatory = false, + ValueFromPipelineByPropertyName = true)] + [ValidateNotNullOrEmpty] + public string ResourceGroupName { get; set; } + + [Parameter( + Position = 1, + Mandatory = true, + ValueFromPipelineByPropertyName = true, + HelpMessage = "Uri to blob")] + [ValidateNotNullOrEmpty] + [Alias("dst")] + public string Destination + { + get; + set; + } + + [Parameter( + Position = 2, + Mandatory = true, + ValueFromPipelineByPropertyName = true, + HelpMessage = "Local path of the vhd file")] + [ValidateNotNullOrEmpty] + [Alias("lf")] + public string LocalFilePath + { + get; + set; + } + + [Parameter( + Position = 3, + Mandatory = false, + ValueFromPipelineByPropertyName = true, + HelpMessage = "Number of uploader threads")] + [ValidateNotNullOrEmpty] + [ValidateRange(1, 64)] + [Alias("th")] + public int? NumberOfUploaderThreads + { + get; + set; + } + + [Parameter( + Position = 4, + Mandatory = false, + ValueFromPipelineByPropertyName = true, + HelpMessage = "Uri to a base image in a blob storage account to apply the difference")] + [ValidateNotNullOrEmpty] + [Alias("bs")] + public string BaseImageUriToPatch + { + get; + set; + } + + [Parameter( + Position = 5, + Mandatory = false, + ValueFromPipelineByPropertyName = true, + ParameterSetName = "Vhd", + HelpMessage = "Delete the blob if already exists")] + [ValidateNotNullOrEmpty] + [Alias("o")] + public SwitchParameter OverWrite + { + get; + set; + } + + public UploadParameters ValidateParameters() + { + BlobUri destinationUri; + if (!BlobUri.TryParseUri(new Uri(Destination), out destinationUri)) + { + throw new ArgumentOutOfRangeException("Destination", this.Destination.ToString()); + } + + BlobUri baseImageUri = null; + if (this.BaseImageUriToPatch != null) + { + if (!BlobUri.TryParseUri(new Uri(BaseImageUriToPatch), out baseImageUri)) + { + throw new ArgumentOutOfRangeException("BaseImageUriToPatch", this.BaseImageUriToPatch.ToString()); + } + + if (!String.IsNullOrEmpty(destinationUri.Uri.Query)) + { + var message = String.Format(Rsrc.ResourceManager.GetString("AddAzureVhdCommandSASUriNotSupportedInPatchMode"), destinationUri.Uri); + throw new ArgumentOutOfRangeException("Destination", message); + } + } + + var storageCredentialsFactory = CreateStorageCredentialsFactory(); + + // TODO: CLU + FileInfo filePath = new FileInfo(LocalFilePath); + /* + PathIntrinsics currentPath = SessionState.Path; + var filePath = new FileInfo(currentPath.GetUnresolvedProviderPathFromPSPath(LocalFilePath.ToString())); + */ + + var parameters = new UploadParameters( + destinationUri, baseImageUri, filePath, OverWrite.IsPresent, + (NumberOfUploaderThreads) ?? DefaultNumberOfUploaderThreads) + { + Cmdlet = this, + BlobObjectFactory = new CloudPageBlobObjectFactory(storageCredentialsFactory, TimeSpan.FromMinutes(1)) + }; + + return parameters; + } + + private StorageCredentialsFactory CreateStorageCredentialsFactory() + { + StorageCredentialsFactory storageCredentialsFactory; + + // TODO: CLU + var storageClient = ClientFactory.CreateArmClient(DefaultProfile.Context, AzureEnvironment.Endpoint.ResourceManager); + /* + var storageClient = AzureSession.ClientFactory.CreateClient( + DefaultProfile.Context, AzureEnvironment.Endpoint.ResourceManager); + */ + + if (StorageCredentialsFactory.IsChannelRequired(new Uri(Destination))) + { + storageCredentialsFactory = new StorageCredentialsFactory(this.ResourceGroupName, storageClient, DefaultContext.Subscription); + } + else + { + storageCredentialsFactory = new StorageCredentialsFactory(); + } + + return storageCredentialsFactory; + } + + protected override void ProcessRecord() + { + var parameters = ValidateParameters(); + var vhdUploadContext = VhdUploaderModel.Upload(parameters); + WriteObject(vhdUploadContext); + } + } +} diff --git a/src/CLU/Microsoft.Azure.Commands.Compute/StorageServices/CloudPageBlobObjectFactory.cs b/src/CLU/Microsoft.Azure.Commands.Compute/StorageServices/CloudPageBlobObjectFactory.cs index 191b0add7960..ace1d244dfb2 100644 --- a/src/CLU/Microsoft.Azure.Commands.Compute/StorageServices/CloudPageBlobObjectFactory.cs +++ b/src/CLU/Microsoft.Azure.Commands.Compute/StorageServices/CloudPageBlobObjectFactory.cs @@ -12,50 +12,51 @@ // limitations under the License. // ---------------------------------------------------------------------------------- -// TODO: Rewrite for removing Sync -//using Microsoft.WindowsAzure.Commands.Sync.Download; -//using Microsoft.WindowsAzure.Commands.Sync.Upload; -//using Microsoft.WindowsAzure.Storage.Blob; -//using Microsoft.WindowsAzure.Storage.RetryPolicies; -//using System; +using Microsoft.WindowsAzure.Commands.Sync.Download; +using Microsoft.WindowsAzure.Commands.Sync.Upload; +using Microsoft.WindowsAzure.Storage.Blob; +using Microsoft.WindowsAzure.Storage.RetryPolicies; +using System; -//namespace Microsoft.Azure.Commands.Compute.StorageServices -//{ -// public class CloudPageBlobObjectFactory : ICloudPageBlobObjectFactory -// { -// private readonly TimeSpan delayBetweenRetries = TimeSpan.FromSeconds(10); -// private readonly StorageCredentialsFactory credentialsFactory; -// private TimeSpan operationTimeout; +namespace Microsoft.Azure.Commands.Compute.StorageServices +{ + public class CloudPageBlobObjectFactory : ICloudPageBlobObjectFactory + { + private readonly TimeSpan delayBetweenRetries = TimeSpan.FromSeconds(10); + private readonly StorageCredentialsFactory credentialsFactory; + private TimeSpan operationTimeout; -// public CloudPageBlobObjectFactory(StorageCredentialsFactory credentialsFactory, TimeSpan operationTimeout) -// { -// this.credentialsFactory = credentialsFactory; -// this.operationTimeout = operationTimeout; -// } + public CloudPageBlobObjectFactory(StorageCredentialsFactory credentialsFactory, TimeSpan operationTimeout) + { + this.credentialsFactory = credentialsFactory; + this.operationTimeout = operationTimeout; + } -// public CloudPageBlob Create(BlobUri destination) -// { -// return new CloudPageBlob(new Uri(destination.BlobPath), credentialsFactory.Create(destination)); -// } + public CloudPageBlob Create(BlobUri destination) + { + return new CloudPageBlob(new Uri(destination.BlobPath), credentialsFactory.Create(destination)); + } -// public bool CreateContainer(BlobUri destination) -// { -// if (String.IsNullOrEmpty(destination.Uri.Query)) -// { -// var destinationBlob = Create(destination); -// return destinationBlob.Container.CreateIfNotExists(this.CreateRequestOptions()); -// } -// return true; -// } + public bool CreateContainer(BlobUri destination) + { + if (String.IsNullOrEmpty(destination.Uri.Query)) + { + var destinationBlob = Create(destination); + // TODO: CLU + return destinationBlob.Container.CreateIfNotExistsAsync(this.CreateRequestOptions(), null).Result; + //return destinationBlob.Container.CreateIfNotExists(this.CreateRequestOptions()); + } + return true; + } -// public BlobRequestOptions CreateRequestOptions() -// { -// return new BlobRequestOptions -// { -// ServerTimeout = this.operationTimeout, -// RetryPolicy = new LinearRetry(delayBetweenRetries, 5) -// }; -// } -// } -//} + public BlobRequestOptions CreateRequestOptions() + { + return new BlobRequestOptions + { + ServerTimeout = this.operationTimeout, + RetryPolicy = new LinearRetry(delayBetweenRetries, 5) + }; + } + } +} diff --git a/src/CLU/Microsoft.Azure.Commands.Compute/StorageServices/SaveAzureVhdCommand.cs b/src/CLU/Microsoft.Azure.Commands.Compute/StorageServices/SaveAzureVhdCommand.cs index 70d146208ce8..a0c93cbccc26 100644 --- a/src/CLU/Microsoft.Azure.Commands.Compute/StorageServices/SaveAzureVhdCommand.cs +++ b/src/CLU/Microsoft.Azure.Commands.Compute/StorageServices/SaveAzureVhdCommand.cs @@ -11,137 +11,139 @@ // See the License for the specific language governing permissions and // limitations under the License. // ---------------------------------------------------------------------------------- -// TODO: Rewrite for removing Sync -//using Microsoft.Azure.Commands.Compute.Common; -//using Microsoft.Azure.Commands.Compute.Models; -//using Microsoft.Azure.Common.Authentication; -//using Microsoft.Azure.Common.Authentication.Models; -//using Microsoft.Azure.Management.Storage; -//using Microsoft.WindowsAzure.Commands.Sync.Download; -//using System; -//using System.IO; -//using System.Management.Automation; - -//namespace Microsoft.Azure.Commands.Compute.StorageServices -//{ -// [Cmdlet(VerbsData.Save, ProfileNouns.Vhd), OutputType(typeof(VhdDownloadContext))] -// public class SaveAzureVhdCommand : ComputeClientBaseCmdlet -// { -// private const int DefaultNumberOfUploaderThreads = 8; -// private const string ResourceGroupParameterSet = "ResourceGroupParameterSetName"; -// private const string StorageKeyParameterSet = "StorageKeyParameterSetName"; - -// [Parameter( -// Position = 0, -// Mandatory = true, -// ParameterSetName = ResourceGroupParameterSet, -// ValueFromPipelineByPropertyName = true)] -// [ValidateNotNullOrEmpty] -// public string ResourceGroupName { get; set; } - -// [Parameter( -// Position = 0, -// Mandatory = true, -// ParameterSetName = StorageKeyParameterSet, -// HelpMessage = "Key of the storage account")] -// [ValidateNotNullOrEmpty] -// [Alias("sk")] -// public string StorageKey { get; set; } - -// [Parameter( -// Position = 1, -// Mandatory = true, -// ValueFromPipelineByPropertyName = true, -// HelpMessage = "Uri to blob")] -// [ValidateNotNullOrEmpty] -// [Alias("src", "Source")] -// public Uri SourceUri { get; set; } - -// [Parameter( -// Position = 2, -// Mandatory = true, -// HelpMessage = "Local path of the vhd file")] -// [ValidateNotNullOrEmpty] -// [Alias("lf")] -// public FileInfo LocalFilePath { get; set; } - -// private int numberOfThreads = DefaultNumberOfUploaderThreads; - -// [Parameter( -// Position = 3, -// Mandatory = false, -// HelpMessage = "Number of downloader threads")] -// [ValidateNotNullOrEmpty] -// [ValidateRange(1, 64)] -// [Alias("th")] -// public int NumberOfThreads -// { -// get { return this.numberOfThreads; } -// set { this.numberOfThreads = value; } -// } - -// [Parameter( -// Position = 4, -// Mandatory = false, -// HelpMessage = "Delete the local file if already exists")] -// [ValidateNotNullOrEmpty] -// [Alias("o")] -// public SwitchParameter OverWrite { get; set; } - -// protected override void ProcessRecord() -// { -// var result = DownloadFromBlobUri( -// this, -// this.SourceUri, -// this.LocalFilePath, -// this.StorageKey, -// this.ResourceGroupName, -// this.NumberOfThreads, -// this.OverWrite); -// WriteObject(result); -// } - - -// private VhdDownloadContext DownloadFromBlobUri( -// ComputeClientBaseCmdlet cmdlet, -// Uri sourceUri, -// FileInfo localFileInfo, -// string storagekey, -// string resourceGroupName, -// int numThreads, -// bool overwrite) -// { -// BlobUri blobUri; -// if (!BlobUri.TryParseUri(sourceUri, out blobUri)) -// { -// throw new ArgumentOutOfRangeException("Source", sourceUri.ToString()); -// } - -// if (storagekey == null) -// { -// var storageClient = AzureSession.ClientFactory.CreateClient( -// DefaultProfile.Context, AzureEnvironment.Endpoint.ResourceManager); - - -// var storageService = storageClient.StorageAccounts.GetProperties(resourceGroupName, blobUri.StorageAccountName); -// if (storageService != null) -// { -// var storageKeys = storageClient.StorageAccounts.ListKeys(resourceGroupName, storageService.StorageAccount.Name); -// storagekey = storageKeys.StorageAccountKeys.Key1; -// } -// } - -// var downloaderParameters = new DownloaderParameters -// { -// BlobUri = blobUri, -// LocalFilePath = localFileInfo.FullName, -// ConnectionLimit = numThreads, -// StorageAccountKey = storagekey, -// ValidateFreeDiskSpace = true, -// OverWrite = overwrite -// }; - -// return VhdDownloaderModel.Download(downloaderParameters, cmdlet); -// } -// } -//} + +using Microsoft.Azure.Commands.Common.Authentication.Models; +using Microsoft.Azure.Commands.Compute.Common; +using Microsoft.Azure.Commands.Compute.Models; +using Microsoft.Azure.Management.Storage; +using Microsoft.WindowsAzure.Commands.Sync.Download; +using System; +using System.IO; +using System.Management.Automation; + +namespace Microsoft.Azure.Commands.Compute.StorageServices +{ + [Cmdlet(VerbsData.Save, ProfileNouns.Vhd), OutputType(typeof(VhdDownloadContext))] + public class SaveAzureVhdCommand : ComputeClientBaseCmdlet + { + private const int DefaultNumberOfUploaderThreads = 8; + private const string ResourceGroupParameterSet = "ResourceGroupParameterSetName"; + private const string StorageKeyParameterSet = "StorageKeyParameterSetName"; + + [Parameter( + Position = 0, + Mandatory = true, + ParameterSetName = ResourceGroupParameterSet, + ValueFromPipelineByPropertyName = true)] + [ValidateNotNullOrEmpty] + public string ResourceGroupName { get; set; } + + [Parameter( + Position = 0, + Mandatory = true, + ParameterSetName = StorageKeyParameterSet, + HelpMessage = "Key of the storage account")] + [ValidateNotNullOrEmpty] + [Alias("sk")] + public string StorageKey { get; set; } + + [Parameter( + Position = 1, + Mandatory = true, + ValueFromPipelineByPropertyName = true, + HelpMessage = "Uri to blob")] + [ValidateNotNullOrEmpty] + [Alias("src", "Source")] + public string SourceUri { get; set; } + + [Parameter( + Position = 2, + Mandatory = true, + HelpMessage = "Local path of the vhd file")] + [ValidateNotNullOrEmpty] + [Alias("lf")] + public string LocalFilePath { get; set; } + + private int numberOfThreads = DefaultNumberOfUploaderThreads; + + [Parameter( + Position = 3, + Mandatory = false, + HelpMessage = "Number of downloader threads")] + [ValidateNotNullOrEmpty] + [ValidateRange(1, 64)] + [Alias("th")] + public int NumberOfThreads + { + get { return this.numberOfThreads; } + set { this.numberOfThreads = value; } + } + + [Parameter( + Position = 4, + Mandatory = false, + HelpMessage = "Delete the local file if already exists")] + [ValidateNotNullOrEmpty] + [Alias("o")] + public SwitchParameter OverWrite { get; set; } + + protected override void ProcessRecord() + { + var result = DownloadFromBlobUri( + this, + new Uri(this.SourceUri), + new FileInfo(this.LocalFilePath), + this.StorageKey, + this.ResourceGroupName, + this.NumberOfThreads, + this.OverWrite); + WriteObject(result); + } + + + private VhdDownloadContext DownloadFromBlobUri( + ComputeClientBaseCmdlet cmdlet, + Uri sourceUri, + FileInfo localFileInfo, + string storagekey, + string resourceGroupName, + int numThreads, + bool overwrite) + { + BlobUri blobUri; + if (!BlobUri.TryParseUri(sourceUri, out blobUri)) + { + throw new ArgumentOutOfRangeException("Source", sourceUri.ToString()); + } + + if (storagekey == null) + { + // TODO: CLU + var storageClient = ClientFactory.CreateArmClient(DefaultProfile.Context, AzureEnvironment.Endpoint.ResourceManager); + /* + var storageClient = AzureSession.ClientFactory.CreateClient( + DefaultProfile.Context, AzureEnvironment.Endpoint.ResourceManager); + */ + + var storageService = storageClient.StorageAccounts.GetProperties(resourceGroupName, blobUri.StorageAccountName); + if (storageService != null) + { + var storageKeys = storageClient.StorageAccounts.ListKeys(resourceGroupName, storageService.Name); + storagekey = storageKeys.Key1; + } + } + + var downloaderParameters = new DownloaderParameters + { + BlobUri = blobUri, + LocalFilePath = localFileInfo.FullName, + ConnectionLimit = numThreads, + StorageAccountKey = storagekey, + ValidateFreeDiskSpace = true, + OverWrite = overwrite + }; + + return VhdDownloaderModel.Download(downloaderParameters, cmdlet); + } + } +} diff --git a/src/CLU/Microsoft.Azure.Commands.Compute/StorageServices/StorageCredentialsFactory.cs b/src/CLU/Microsoft.Azure.Commands.Compute/StorageServices/StorageCredentialsFactory.cs index 1e0b508f3b68..372487f07d3d 100644 --- a/src/CLU/Microsoft.Azure.Commands.Compute/StorageServices/StorageCredentialsFactory.cs +++ b/src/CLU/Microsoft.Azure.Commands.Compute/StorageServices/StorageCredentialsFactory.cs @@ -12,53 +12,51 @@ // limitations under the License. // ---------------------------------------------------------------------------------- -// TODO: Rewrite for removing Sync -//using Microsoft.Azure.Commands.Common.Authentication.Models; -//using Microsoft.Azure.Common.Authentication.Models; -//using Microsoft.Azure.Management.Storage; -//using Microsoft.WindowsAzure.Commands.Sync.Download; -//using Microsoft.WindowsAzure.Storage.Auth; -//using System; -//using Rsrc = Microsoft.Azure.Commands.Compute.Properties.Resources; +using Microsoft.Azure.Commands.Common.Authentication.Models; +using Microsoft.Azure.Management.Storage; +using Microsoft.WindowsAzure.Commands.Sync.Download; +using Microsoft.WindowsAzure.Storage.Auth; +using System; +using Rsrc = Microsoft.Azure.Commands.Compute.Properties.Resources; -//namespace Microsoft.Azure.Commands.Compute.StorageServices -//{ -// public class StorageCredentialsFactory -// { -// private StorageManagementClient client; -// private AzureSubscription currentSubscription; -// public string resourceGroupName { get; set; } +namespace Microsoft.Azure.Commands.Compute.StorageServices +{ + public class StorageCredentialsFactory + { + private StorageManagementClient client; + private AzureSubscription currentSubscription; + public string resourceGroupName { get; set; } -// public static bool IsChannelRequired(Uri destination) -// { -// return String.IsNullOrEmpty(destination.Query); -// } + public static bool IsChannelRequired(Uri destination) + { + return String.IsNullOrEmpty(destination.Query); + } -// public StorageCredentialsFactory() -// { -// this.resourceGroupName = null; -// } + public StorageCredentialsFactory() + { + this.resourceGroupName = null; + } -// public StorageCredentialsFactory(string resourceGroupName, StorageManagementClient client, AzureSubscription currentSubscription) -// { -// this.resourceGroupName = resourceGroupName; -// this.client = client; -// this.currentSubscription = currentSubscription; -// } + public StorageCredentialsFactory(string resourceGroupName, StorageManagementClient client, AzureSubscription currentSubscription) + { + this.resourceGroupName = resourceGroupName; + this.client = client; + this.currentSubscription = currentSubscription; + } -// public StorageCredentials Create(BlobUri destination) -// { -// if (IsChannelRequired(destination.Uri)) -// { -// if(currentSubscription == null) -// { -// throw new ArgumentException(Rsrc.StorageCredentialsFactoryCurrentSubscriptionNotSet, "SubscriptionId"); -// } -// var storageKeys = this.client.StorageAccounts.ListKeys(this.resourceGroupName, destination.StorageAccountName); -// return new StorageCredentials(destination.StorageAccountName, storageKeys.StorageAccountKeys.Key1); -// } + public StorageCredentials Create(BlobUri destination) + { + if (IsChannelRequired(destination.Uri)) + { + if (currentSubscription == null) + { + throw new ArgumentException(Rsrc.ResourceManager.GetString("StorageCredentialsFactoryCurrentSubscriptionNotSet"), "SubscriptionId"); + } + var storageKeys = this.client.StorageAccounts.ListKeys(this.resourceGroupName, destination.StorageAccountName); + return new StorageCredentials(destination.StorageAccountName, storageKeys.Key1); + } -// return new StorageCredentials(destination.Uri.Query); -// } -// } -//} + return new StorageCredentials(destination.Uri.Query); + } + } +} diff --git a/src/CLU/Microsoft.Azure.Commands.Compute/project.json b/src/CLU/Microsoft.Azure.Commands.Compute/project.json index d6f238929066..a32d23a4f86a 100644 --- a/src/CLU/Microsoft.Azure.Commands.Compute/project.json +++ b/src/CLU/Microsoft.Azure.Commands.Compute/project.json @@ -53,7 +53,9 @@ "Microsoft.Azure.Management.Compute": "11.0.0-prerelease", "Microsoft.Azure.Management.Network": "3.0.3-preview", "Microsoft.Azure.Management.Storage": "4.0.0-preview", - "AutoMapper": "4.1.1" + "AutoMapper": "4.1.1", + "VhdManagement": "1.0.0-*", + "Sync": "1.0.0-*" }, "compilationOptions": {"emitEntryPoint": true} } diff --git a/src/CLU/Sync/ComputeStats.cs b/src/CLU/Sync/ComputeStats.cs new file mode 100644 index 000000000000..f11712505458 --- /dev/null +++ b/src/CLU/Sync/ComputeStats.cs @@ -0,0 +1,51 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System.Collections.Generic; + +namespace Microsoft.WindowsAzure.Commands.Sync +{ + public class ComputeStats + { + IList history; + int historySize; + + public ComputeStats() : this(60) + { + } + + public ComputeStats(int historySize) + { + history = new List(historySize); + this.historySize = historySize; + } + + public double ComputeAvg(double current) + { + if (history.Count > historySize) + { + history.RemoveAt(0); + } + history.Add(current); + double sum = 0.0; + foreach (var x in history) + { + sum += x; + } + return sum / history.Count; + } + } +} + + diff --git a/src/CLU/Sync/Download/BlobHandle.cs b/src/CLU/Sync/Download/BlobHandle.cs new file mode 100644 index 000000000000..ac247519e1f1 --- /dev/null +++ b/src/CLU/Sync/Download/BlobHandle.cs @@ -0,0 +1,98 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Microsoft.WindowsAzure.Commands.Sync.Upload; +using Microsoft.WindowsAzure.Commands.Tools.Vhd.Model; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Auth; +using Microsoft.WindowsAzure.Storage.Blob; +using Microsoft.WindowsAzure.Storage.RetryPolicies; + +namespace Microsoft.WindowsAzure.Commands.Sync.Download +{ + public class BlobHandle + { + const int MegaByte = 1024 * 1024; + + private readonly BlobUri blobUri; + private readonly string storageAccountKey; + private readonly CloudBlobContainer container; + private readonly BlobRequestOptions blobRequestOptions; + private readonly CloudPageBlob pageBlob; + + public BlobHandle(BlobUri blobUri, string storageAccountKey) + { + this.blobUri = blobUri; + this.storageAccountKey = storageAccountKey; + var blobClient = new CloudBlobClient(new Uri(this.blobUri.BaseUri), new StorageCredentials(this.blobUri.StorageAccountName, this.storageAccountKey)); + this.container = blobClient.GetContainerReference(this.blobUri.BlobContainerName); + // TODO: CLU + //this.container.FetchAttributes(); + this.container.FetchAttributesAsync().Wait(); + this.pageBlob = this.container.GetPageBlobReference(blobUri.BlobName); + this.blobRequestOptions = new BlobRequestOptions + { + ServerTimeout = TimeSpan.FromMinutes(5), + RetryPolicy = new LinearRetry(TimeSpan.FromMinutes(1), 3) + }; + + // TODO: CLU + //this.pageBlob.FetchAttributes(new AccessCondition(), blobRequestOptions); + this.pageBlob.FetchAttributesAsync(new AccessCondition(), blobRequestOptions, null).Wait(); + } + + public CloudPageBlob Blob { get { return this.pageBlob; } } + + public IEnumerable GetEmptyRanges() + { + var blobRange = new List {IndexRange.FromLength(0, this.Length)}; + return IndexRange.SubstractRanges(blobRange, GetPageRanges()); + } + + public IEnumerable GetUploadableRanges() + { + IEnumerable ranges = GetPageRanges(); + ranges = Enumerable.ToList(IndexRangeHelper.ChunkRangesBySize(ranges, 2 * MegaByte)); + return ranges; + } + + private IEnumerable GetPageRanges() + { + // TODO: CLU + //pageBlob.FetchAttributes(new AccessCondition(), blobRequestOptions); + pageBlob.FetchAttributesAsync(new AccessCondition(), blobRequestOptions, null).Wait(); + // TODO: CLU + //IEnumerable pageRanges = pageBlob.GetPageRanges(null, null, new AccessCondition(), blobRequestOptions); + IEnumerable pageRanges = pageBlob.GetPageRangesAsync(null, null, new AccessCondition(), blobRequestOptions, null).Result; + pageRanges.OrderBy(range => range.StartOffset); + return pageRanges.Select(pr => new IndexRange(pr.StartOffset, pr.EndOffset)); + } + + public Stream OpenStream() + { + // TODO: CLU + //return this.container.GetPageBlobReference(blobUri.BlobName).OpenRead(new AccessCondition(), blobRequestOptions); + return this.container.GetPageBlobReference(blobUri.BlobName).OpenReadAsync(new AccessCondition(), blobRequestOptions, null).Result; + } + + public long Length + { + get { return pageBlob.Properties.Length; } + } + } +} \ No newline at end of file diff --git a/src/CLU/Sync/Download/BlobUri.cs b/src/CLU/Sync/Download/BlobUri.cs new file mode 100644 index 000000000000..660e062c53bf --- /dev/null +++ b/src/CLU/Sync/Download/BlobUri.cs @@ -0,0 +1,576 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Collections; +using System.Collections.Specialized; +using System.Runtime.Serialization; +using System.Text; + +namespace Microsoft.WindowsAzure.Commands.Sync.Download +{ + public class BlobUri + { + public static bool TryParseUri(Uri uri, out BlobUri blobUri) + { + blobUri = null; + string storageAccountName; + string storageDomainName; + string blobContainerName; + string blobName; + string queryString; + string secret; + + var result = TryParseUri(uri, + out storageAccountName, + out storageDomainName, + out blobContainerName, + out blobName, + out queryString, + out secret); + + if (!result) + { + return false; + } + blobUri = new BlobUri(uri, storageAccountName, storageDomainName, blobContainerName, blobName, queryString); + return true; + } + + internal static bool TryParseUri(Uri blobUri, + out string storageAccountName, + out string storageDomainName, + out string blobContainerName, + out string blobName, + out string queryString, + out string secret) + { + storageAccountName = null; + storageDomainName = null; + blobContainerName = null; + blobName = null; + secret = null; + queryString = null; + + string[] hostSegments = blobUri.DnsSafeHost.ToLower().Split('.'); + + if (hostSegments.Length < 2) + { + return false; + } + + storageAccountName = hostSegments[0]; + storageDomainName = string.Join(".", hostSegments, 1, hostSegments.Length - 1); + + blobContainerName = null; + blobName = null; + + string[] segments = blobUri.AbsolutePath.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries); + + if (segments.Length < 2) + { + return false; //Must have atleast a containerName and BlobName + } + + blobName = HttpUtility.UrlDecode( + string.Join( + "/", segments, 1, segments.Length - 1)); + blobContainerName = HttpUtility.UrlDecode(segments[0]); + + NameValueCollection queryValues = HttpUtility.ParseQueryString(blobUri.Query); + + StringBuilder queryBuilder = new StringBuilder(); + bool firstQuery = true; + + foreach (string key in queryValues.AllKeys) + { + if (string.Equals(key, "dsas_secret", StringComparison.OrdinalIgnoreCase)) + { + secret = queryValues["dsas_secret"]; + } + else + { + if (firstQuery) + { + queryBuilder.AppendFormat("?{0}={1}", key.Replace("?", ""), queryValues[key]); + firstQuery = false; + } + else + { + queryBuilder.AppendFormat("&{0}={1}", key, queryValues[key]); + } + } + } + + if (!string.IsNullOrEmpty(secret)) + { + //TODO[JWA], Find out the complete set of decoding; + secret = secret.Replace(' ', '+'); + } + + queryString = queryBuilder.ToString(); + return true; + } + + public Uri Uri { get; private set; } + public string BaseUri { get; private set; } + public string StorageAccountName { get; private set; } + public string StorageDomainName { get; private set; } + public string BlobContainerName { get; private set; } + public string BlobName { get; private set; } + public string QueryString { get; set; } + public string BlobPath { get; private set; } + + public BlobUri(Uri uri, string storageAccountName, string storageDomainName, string blobContainerName, string blobName, string queryString) + { + Uri = uri; + StorageAccountName = storageAccountName; + StorageDomainName = storageDomainName; + BlobContainerName = blobContainerName; + BlobName = blobName; + QueryString = queryString; + // TODO: CLU + BaseUri = uri.Scheme + "://" + uri.DnsSafeHost; + //BaseUri = uri.Scheme + Uri.SchemeDelimiter + uri.DnsSafeHost; + BlobPath = this.BaseUri + uri.LocalPath; + } + + } + +#region HttpUtility port from .Net 4.0 + + class HttpUtility + { + public static string UrlDecode(string str) + { + if (str == null) + { + return null; + } + return UrlDecode(str, Encoding.UTF8); + } + + public static string UrlDecode(string str, Encoding e) + { + if (str == null) + { + return null; + } + return UrlDecodeStringFromStringInternal(str, e); + } + + private static string UrlDecodeStringFromStringInternal(string s, Encoding e) + { + int length = s.Length; + UrlDecoder decoder = new UrlDecoder(length, e); + for (int i = 0; i < length; i++) + { + char ch = s[i]; + if (ch == '+') + { + ch = ' '; + } + else if ((ch == '%') && (i < (length - 2))) + { + if ((s[i + 1] == 'u') && (i < (length - 5))) + { + int num3 = HexToInt(s[i + 2]); + int num4 = HexToInt(s[i + 3]); + int num5 = HexToInt(s[i + 4]); + int num6 = HexToInt(s[i + 5]); + if (((num3 < 0) || (num4 < 0)) || ((num5 < 0) || (num6 < 0))) + { + goto Label_0106; + } + ch = (char)((((num3 << 12) | (num4 << 8)) | (num5 << 4)) | num6); + i += 5; + decoder.AddChar(ch); + continue; + } + int num7 = HexToInt(s[i + 1]); + int num8 = HexToInt(s[i + 2]); + if ((num7 >= 0) && (num8 >= 0)) + { + byte b = (byte)((num7 << 4) | num8); + i += 2; + decoder.AddByte(b); + continue; + } + } + Label_0106: + if ((ch & 0xff80) == 0) + { + decoder.AddByte((byte)ch); + } + else + { + decoder.AddChar(ch); + } + } + return decoder.GetString(); + } + + private static int HexToInt(char h) + { + if ((h >= '0') && (h <= '9')) + { + return (h - '0'); + } + if ((h >= 'a') && (h <= 'f')) + { + return ((h - 'a') + 10); + } + if ((h >= 'A') && (h <= 'F')) + { + return ((h - 'A') + 10); + } + return -1; + } + + public static string UrlEncodeUnicode(string str) + { + if (str == null) + { + return null; + } + return UrlEncodeUnicodeStringToStringInternal(str, false); + } + + private static string UrlEncodeUnicodeStringToStringInternal(string s, bool ignoreAscii) + { + int length = s.Length; + StringBuilder builder = new StringBuilder(length); + for (int i = 0; i < length; i++) + { + char ch = s[i]; + if ((ch & 0xff80) == 0) + { + if (ignoreAscii || IsSafe(ch)) + { + builder.Append(ch); + } + else if (ch == ' ') + { + builder.Append('+'); + } + else + { + builder.Append('%'); + builder.Append(IntToHex((ch >> 4) & '\x000f')); + builder.Append(IntToHex(ch & '\x000f')); + } + } + else + { + builder.Append("%u"); + builder.Append(IntToHex((ch >> 12) & '\x000f')); + builder.Append(IntToHex((ch >> 8) & '\x000f')); + builder.Append(IntToHex((ch >> 4) & '\x000f')); + builder.Append(IntToHex(ch & '\x000f')); + } + } + return builder.ToString(); + } + + internal static bool IsSafe(char ch) + { + if ((((ch >= 'a') && (ch <= 'z')) || ((ch >= 'A') && (ch <= 'Z'))) || ((ch >= '0') && (ch <= '9'))) + { + return true; + } + switch (ch) + { + case '\'': + case '(': + case ')': + case '*': + case '-': + case '.': + case '_': + case '!': + return true; + } + return false; + } + + internal static char IntToHex(int n) + { + if (n <= 9) + { + return (char)(n + 0x30); + } + return (char)((n - 10) + 0x61); + } + + public static NameValueCollection ParseQueryString(string query) + { + return ParseQueryString(query, Encoding.UTF8); + } + + public static NameValueCollection ParseQueryString(string query, Encoding encoding) + { + if (query == null) + { + throw new ArgumentNullException("query"); + } + if (encoding == null) + { + throw new ArgumentNullException("encoding"); + } + if ((query.Length > 0) && (query[0] == '?')) + { + query = query.Substring(1); + } + return new HttpValueCollection(query, false, true, encoding); + } + + private class UrlDecoder + { + // Fields + private int _bufferSize; + private byte[] _byteBuffer; + private char[] _charBuffer; + private Encoding _encoding; + private int _numBytes; + private int _numChars; + + // Methods + internal UrlDecoder(int bufferSize, Encoding encoding) + { + this._bufferSize = bufferSize; + this._encoding = encoding; + this._charBuffer = new char[bufferSize]; + } + + internal void AddByte(byte b) + { + if (this._byteBuffer == null) + { + this._byteBuffer = new byte[this._bufferSize]; + } + this._byteBuffer[this._numBytes++] = b; + } + + internal void AddChar(char ch) + { + if (this._numBytes > 0) + { + this.FlushBytes(); + } + this._charBuffer[this._numChars++] = ch; + } + + private void FlushBytes() + { + if (this._numBytes > 0) + { + this._numChars += this._encoding.GetChars(this._byteBuffer, 0, this._numBytes, this._charBuffer, this._numChars); + this._numBytes = 0; + } + } + + internal string GetString() + { + if (this._numBytes > 0) + { + this.FlushBytes(); + } + if (this._numChars > 0) + { + return new string(this._charBuffer, 0, this._numChars); + } + return string.Empty; + } + } + + } + + // TODO: CLU + //[Serializable] + internal class HttpValueCollection : NameValueCollection + { + // Methods + internal HttpValueCollection() + : base(StringComparer.OrdinalIgnoreCase) + { + } + + internal HttpValueCollection(int capacity) + : base(capacity, (IEqualityComparer)StringComparer.OrdinalIgnoreCase) + { + } + + // TODO: CLU + /* + protected HttpValueCollection(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } + */ + + internal HttpValueCollection(string str, bool readOnly, bool urlencoded, Encoding encoding) + : base(StringComparer.OrdinalIgnoreCase) + { + if (!string.IsNullOrEmpty(str)) + { + this.FillFromString(str, urlencoded, encoding); + } + base.IsReadOnly = readOnly; + } + + + internal void FillFromString(string s) + { + this.FillFromString(s, false, null); + } + + internal void FillFromString(string s, bool urlencoded, Encoding encoding) + { + int num = (s != null) ? s.Length : 0; + for (int i = 0; i < num; i++) + { + int startIndex = i; + int num4 = -1; + while (i < num) + { + char ch = s[i]; + if (ch == '=') + { + if (num4 < 0) + { + num4 = i; + } + } + else if (ch == '&') + { + break; + } + i++; + } + string str = null; + string str2 = null; + if (num4 >= 0) + { + str = s.Substring(startIndex, num4 - startIndex); + str2 = s.Substring(num4 + 1, (i - num4) - 1); + } + else + { + str2 = s.Substring(startIndex, i - startIndex); + } + if (urlencoded) + { + base.Add(HttpUtility.UrlDecode(str, encoding), HttpUtility.UrlDecode(str2, encoding)); + } + else + { + base.Add(str, str2); + } + if ((i == (num - 1)) && (s[i] == '&')) + { + base.Add(null, string.Empty); + } + } + } + + internal void MakeReadOnly() + { + base.IsReadOnly = true; + } + + internal void MakeReadWrite() + { + base.IsReadOnly = false; + } + + internal void Reset() + { + base.Clear(); + } + + public override string ToString() + { + return this.ToString(true); + } + + internal virtual string ToString(bool urlencoded) + { + return this.ToString(urlencoded, null); + } + + internal virtual string ToString(bool urlencoded, IDictionary excludeKeys) + { + int count = this.Count; + if (count == 0) + { + return string.Empty; + } + StringBuilder builder = new StringBuilder(); + bool flag = (excludeKeys != null) && (excludeKeys["__VIEWSTATE"] != null); + for (int i = 0; i < count; i++) + { + string key = this.GetKey(i); + if (((!flag || (key == null)) || !key.StartsWith("__VIEWSTATE", StringComparison.Ordinal)) && (((excludeKeys == null) || (key == null)) || (excludeKeys[key] == null))) + { + string str3; + if (urlencoded) + { + key = HttpUtility.UrlEncodeUnicode(key); + } + string str2 = !string.IsNullOrEmpty(key) ? (key + "=") : string.Empty; + ArrayList list = (ArrayList)base.BaseGet(i); + int num3 = (list != null) ? list.Count : 0; + if (builder.Length > 0) + { + builder.Append('&'); + } + if (num3 == 1) + { + builder.Append(str2); + str3 = (string)list[0]; + if (urlencoded) + { + str3 = HttpUtility.UrlEncodeUnicode(str3); + } + builder.Append(str3); + } + else if (num3 == 0) + { + builder.Append(str2); + } + else + { + for (int j = 0; j < num3; j++) + { + if (j > 0) + { + builder.Append('&'); + } + builder.Append(str2); + str3 = (string)list[j]; + if (urlencoded) + { + str3 = HttpUtility.UrlEncodeUnicode(str3); + } + builder.Append(str3); + } + } + } + } + return builder.ToString(); + } + } +#endregion + +} \ No newline at end of file diff --git a/src/CLU/Sync/Download/Downloader.cs b/src/CLU/Sync/Download/Downloader.cs new file mode 100644 index 000000000000..53503a1e00ac --- /dev/null +++ b/src/CLU/Sync/Download/Downloader.cs @@ -0,0 +1,171 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.ServiceModel.Channels; +using Microsoft.WindowsAzure.Commands.Sync.Threading; +using Microsoft.WindowsAzure.Commands.Tools.Vhd.Model; + +namespace Microsoft.WindowsAzure.Commands.Sync.Download +{ + public class Downloader + { + private const int DefaultConnectionLimit = 24; + private DownloaderParameters parameters; + + public Downloader(BlobUri blobUri, string storageAccountKey, string locaFilePath) + { + this.parameters = new DownloaderParameters + { + BlobUri = blobUri, + LocalFilePath = locaFilePath, + ConnectionLimit = DefaultConnectionLimit, + StorageAccountKey = storageAccountKey, + ValidateFreeDiskSpace = false, + ProgressDownloadStatus = Program.SyncOutput.ProgressDownloadStatus, + ProgressDownloadComplete = Program.SyncOutput.ProgressDownloadComplete + }; + } + + public Downloader(DownloaderParameters parameters) + { + this.parameters = parameters; + } + + public void Download() + { + if(parameters.OverWrite) + { + DeleteTempVhdIfExist(parameters.LocalFilePath); + } + else + { + if (File.Exists(parameters.LocalFilePath)) + { + var message = String.Format("File already exists, you can use Overwrite option to delete it:'{0}'", parameters.LocalFilePath); + throw new ArgumentException(message); + } + } + + var blobHandle = new BlobHandle(parameters.BlobUri, this.parameters.StorageAccountKey); + + if (parameters.ValidateFreeDiskSpace) + { + TryValidateFreeDiskSpace(parameters.LocalFilePath, blobHandle.Length); + } + + const int megaByte = 1024 * 1024; + + var ranges = blobHandle.GetUploadableRanges(); + var bufferManager = BufferManager.CreateBufferManager(Int32.MaxValue, 20 * megaByte); + var downloadStatus = new ProgressStatus(0, ranges.Sum(r => r.Length), new ComputeStats()); + + Trace.WriteLine(String.Format("Total Data:{0}", ranges.Sum(r => r.Length))); + + Program.SyncOutput.WriteVerboseWithTimestamp("Downloading the blob: {0}", parameters.BlobUri.BlobName); + + var fileStreamLock = new object(); + using (new ServicePointHandler(parameters.BlobUri.Uri, parameters.ConnectionLimit)) + { + using (new ProgressTracker(downloadStatus, parameters.ProgressDownloadStatus, parameters.ProgressDownloadComplete, TimeSpan.FromSeconds(1))) + { + using (var fileStream = new FileStream(parameters.LocalFilePath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Write, 8 * megaByte, FileOptions.WriteThrough)) + { + fileStream.SetLength(0); + fileStream.SetLength(blobHandle.Length); + + LoopResult lr = Parallel.ForEach(ranges, + blobHandle.OpenStream, + (r, b) => + { + b.Seek(r.StartIndex, SeekOrigin.Begin); + + byte[] buffer = this.EnsureReadAsSize(b, (int)r.Length, bufferManager); + + lock (fileStreamLock) + { + Trace.WriteLine(String.Format("Range:{0}", r)); + fileStream.Seek(r.StartIndex, SeekOrigin.Begin); + fileStream.Write(buffer, 0, (int)r.Length); + fileStream.Flush(); + } + + downloadStatus.AddToProcessedBytes((int)r.Length); + }, + pbwlf => + { + pbwlf.Dispose(); + }, + parameters.ConnectionLimit); + + if (lr.IsExceptional) + { + throw new AggregateException(lr.Exceptions); + } + } + } + } + Program.SyncOutput.WriteVerboseWithTimestamp("Blob downloaded successfullty: {0}", parameters.BlobUri.BlobName); + } + + private void TryValidateFreeDiskSpace(string destination, long blobLength) + { + try + { + // TODO: CLU + /* + DriveInfo info = new DriveInfo(destination); + if(info.AvailableFreeSpace < blobLength) + { + string message = String.Format("Insufficient disk space: Blob's size is {0}, however available space is {1}.", blobLength, info.AvailableFreeSpace); + throw new ArgumentOutOfRangeException(message); + } + */ + } + catch (Exception) + { + } + } + + private void DeleteTempVhdIfExist(string fileName) + { + if (File.Exists(fileName)) + File.Delete(fileName); + } + + private byte[] EnsureReadAsSize(Stream stream, int size, BufferManager manager) + { + byte[] buffer = manager.TakeBuffer(size); + int byteRead = 0; + int totalRead = 0; + int sizeLeft = size; + do + { + byteRead = stream.Read(buffer, totalRead, sizeLeft); + totalRead += byteRead; + if (totalRead == size) + { + break; + } + + sizeLeft = sizeLeft - byteRead; + } while (true); + + return buffer; + } + } +} diff --git a/src/CLU/Sync/Download/DownloaderParameters.cs b/src/CLU/Sync/Download/DownloaderParameters.cs new file mode 100644 index 000000000000..cb835753c7f7 --- /dev/null +++ b/src/CLU/Sync/Download/DownloaderParameters.cs @@ -0,0 +1,30 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.WindowsAzure.Commands.Sync.Download +{ + public class DownloaderParameters + { + public BlobUri BlobUri { get; set; } + public string LocalFilePath { get; set; } + public int ConnectionLimit { get; set; } + public string StorageAccountKey { get; set; } + public bool ValidateFreeDiskSpace { get; set; } + public bool OverWrite { get; set; } + public Action ProgressDownloadStatus { get; set; } + public Action ProgressDownloadComplete { get; set; } + } +} \ No newline at end of file diff --git a/src/CLU/Sync/EntryStub.cs b/src/CLU/Sync/EntryStub.cs new file mode 100644 index 000000000000..1a983b989ed1 --- /dev/null +++ b/src/CLU/Sync/EntryStub.cs @@ -0,0 +1,24 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace Microsoft.WindowsAzure.Commands.Sync +{ + public class EntryStub + { + public static void Main(string[] args) + { + // empty entry point + } + } +} diff --git a/src/CLU/Sync/IO/StreamWithReadProgress.cs b/src/CLU/Sync/IO/StreamWithReadProgress.cs new file mode 100644 index 000000000000..9e86ed1fe2fb --- /dev/null +++ b/src/CLU/Sync/IO/StreamWithReadProgress.cs @@ -0,0 +1,107 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.IO; + +namespace Microsoft.WindowsAzure.Commands.Sync.IO +{ + internal class StreamWithReadProgress : Stream + { + private readonly Stream innerStream; + private readonly TimeSpan progressInterval; + private ProgressStatus readStatus; + private ProgressTracker progressTracker; + + public StreamWithReadProgress(Stream innerStream, TimeSpan progressInterval) + { + this.innerStream = innerStream; + this.progressInterval = progressInterval; + this.readStatus = new ProgressStatus(0, this.innerStream.Length, new ComputeStats()); + + this.progressTracker = new ProgressTracker(this.readStatus, + Program.SyncOutput.ProgressOperationStatus, + Program.SyncOutput.ProgressOperationComplete, + this.progressInterval); + } + + public override void Flush() + { + this.innerStream.Flush(); + } + + public override long Seek(long offset, SeekOrigin origin) + { + return this.innerStream.Seek(offset, origin); + } + + public override void SetLength(long value) + { + this.innerStream.SetLength(value); + } + + public override int Read(byte[] buffer, int offset, int count) + { + var readCount = this.innerStream.Read(buffer, offset, count); + readStatus.AddToProcessedBytes(readCount); + return readCount; + } + + public override void Write(byte[] buffer, int offset, int count) + { + this.innerStream.Write(buffer, offset, count); + } + + public override bool CanRead + { + get { return this.innerStream.CanRead; } + } + + public override bool CanSeek + { + get { return this.innerStream.CanSeek; } + } + + public override bool CanWrite + { + get { return this.innerStream.CanWrite; } + } + + public override long Length + { + get { return this.innerStream.Length; } + } + + public override long Position + { + get { return this.innerStream.Position; } + set { this.innerStream.Position = value; } + } + + // TODO: CLU + /* + public override void Close() + { + this.progressTracker.Dispose(); + this.innerStream.Close(); + } + */ + + protected override void Dispose(bool disposing) + { + this.progressTracker.Dispose(); + this.innerStream.Dispose(); + } + } +} \ No newline at end of file diff --git a/src/CLU/Sync/MSSharedLibKey.snk b/src/CLU/Sync/MSSharedLibKey.snk new file mode 100644 index 000000000000..695f1b38774e Binary files /dev/null and b/src/CLU/Sync/MSSharedLibKey.snk differ diff --git a/src/CLU/Sync/Program.cs b/src/CLU/Sync/Program.cs new file mode 100644 index 000000000000..b8a3ecb0f5da --- /dev/null +++ b/src/CLU/Sync/Program.cs @@ -0,0 +1,80 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Microsoft.WindowsAzure.Commands.Tools.Vhd.Model; + +namespace Microsoft.WindowsAzure.Commands.Sync +{ + public class Program + { + static public ISyncOutputEvents SyncOutput + { + get + { + return RawEvents; + } + set + { + if (value == null) + { + throw new ArgumentNullException("value"); + } + RawEvents = value; + } + } +// static public IProgramConfiguration Configuration { get; private set; } + //private static ISyncOutputEvents RawEvents = new SyncOutputEvents(); + private static ISyncOutputEvents RawEvents = null; + } + + public interface ISyncOutputEvents + { + void MessageCreatingNewPageBlob(long pageBlobSize); + void MessageResumingUpload(); + void ErrorUploadFailedWithExceptions(IList exjceptions); + + + void ProgressCopyComplete(TimeSpan elapsed); + void ProgressCopyStatus(double precentComplete, double avgThroughputMbps, TimeSpan remainingTime); + void ProgressCopyStatus(ProgressRecord record); + + void ProgressUploadStatus(ProgressRecord record); + void ProgressUploadStatus(double precentComplete, double avgThroughputMbps, TimeSpan remainingTime); + void ProgressUploadComplete(TimeSpan elapsed); + + void ProgressDownloadStatus(ProgressRecord record); + void ProgressDownloadStatus(double precentComplete, double avgThroughputMbps, TimeSpan remainingTime); + void ProgressDownloadComplete(TimeSpan elapsed); + + void ProgressOperationStatus(ProgressRecord record); + void ProgressOperationStatus(double percentComplete, double avgThroughputMbps, TimeSpan remainingTime); + void ProgressOperationComplete(TimeSpan elapsed); + + void MessageCalculatingMD5Hash(string filePath); + void MessageMD5HashCalculationFinished(); + + void MessageRetryingAfterANetworkDisruption(); + void DebugRetryingAfterException(Exception lastException); + + void MessageDetectingActualDataBlocks(); + void MessageDetectingActualDataBlocksCompleted(); + void MessagePrintBlockRange(IndexRange range); + void DebugEmptyBlockDetected(IndexRange range); + void ProgressEmptyBlockDetection(int processedRangeCount, int totalRangeCount); + + void WriteVerboseWithTimestamp(string message, params object[] args); + } +} diff --git a/src/CLU/Sync/ProgressRecord.cs b/src/CLU/Sync/ProgressRecord.cs new file mode 100644 index 000000000000..92940867a2d4 --- /dev/null +++ b/src/CLU/Sync/ProgressRecord.cs @@ -0,0 +1,25 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.WindowsAzure.Commands.Sync +{ + public class ProgressRecord + { + public double PercentComplete { get; set; } + public double AvgThroughputMbPerSecond { get; set; } + public TimeSpan RemainingTime { get; set; } + } +} \ No newline at end of file diff --git a/src/CLU/Sync/ProgressStatus.cs b/src/CLU/Sync/ProgressStatus.cs new file mode 100644 index 000000000000..94efc80cf81b --- /dev/null +++ b/src/CLU/Sync/ProgressStatus.cs @@ -0,0 +1,104 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.WindowsAzure.Commands.Sync +{ + public class ProgressStatus + { + const double MB = 1024.0 * 1024.0; + readonly object thisLock = new object(); + + public ProgressStatus(long alreadyProcessedBytes, long totalLength) : this(alreadyProcessedBytes, totalLength, new ComputeStats()) + { + } + + public ProgressStatus(long alreadyProcessedBytes, long totalLength, ComputeStats computeStats) + { + this.PreExistingBytes = alreadyProcessedBytes; + this.BytesProcessed = alreadyProcessedBytes; + this.TotalLength = totalLength; + this.ThrougputStats = computeStats; + this.StartTime = DateTime.UtcNow; + } + + long PreExistingBytes { get; set; } + internal long BytesProcessed { get; private set; } + long TotalLength { get; set; } + DateTime StartTime { get; set; } + ComputeStats ThrougputStats { get; set; } + + public bool TryGetProgressRecord(out ProgressRecord record) + { + record = null; + lock (thisLock) + { + if (HasProgess()) + { + record = Progress(); + return true; + } + } + return false; + } + + public void AddToProcessedBytes(long size) + { + lock (thisLock) + { + this.BytesProcessed += size; + } + } + + bool HasProgess() + { + return this.BytesProcessed > this.PreExistingBytes; + } + + ProgressRecord Progress() + { + double computeAvg = ThrougputStats.ComputeAvg(ThroughputMBs()); + double avtThroughputMbps = 8.0 * computeAvg; + double remainingSeconds = (RemainingMB() / computeAvg); + var pr = new ProgressRecord + { + PercentComplete = PercentComplete(), + AvgThroughputMbPerSecond = avtThroughputMbps, + RemainingTime = TimeSpan.FromSeconds(remainingSeconds) + }; + return pr; + } + + double RemainingMB() + { + return (this.TotalLength - this.BytesProcessed) / MB; + } + + double ThroughputMBs() + { + return (this.BytesProcessed - this.PreExistingBytes) / MB / ProcessTime().TotalSeconds; + } + + TimeSpan ProcessTime() + { + return DateTime.UtcNow - this.StartTime; + } + + double PercentComplete() + { + return 100.0 * this.BytesProcessed / ((double)this.TotalLength); + } + } +} \ No newline at end of file diff --git a/src/CLU/Sync/ProgressTracker.cs b/src/CLU/Sync/ProgressTracker.cs new file mode 100644 index 000000000000..32cf775d6acf --- /dev/null +++ b/src/CLU/Sync/ProgressTracker.cs @@ -0,0 +1,144 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Diagnostics; +// TODO: CLU +using System.Threading; +//using System.Timers; + +namespace Microsoft.WindowsAzure.Commands.Sync +{ + public class ProgressTracker : IDisposable + { + private readonly ProgressStatus progressStatus; + private readonly Action progress; + private readonly Action complete; + private readonly TimeSpan progressInterval; + private Timer progressTimer; + private object thisLock = new object(); + // TODO: CLU + private TimerCallback progressTimerOnElapsed; + //private ElapsedEventHandler progressTimerOnElapsed; + private Stopwatch stopWatch; + private bool isDisposed; + + public ProgressTracker(ProgressStatus progressStatus) : + this(progressStatus, Program.SyncOutput.ProgressUploadStatus, Program.SyncOutput.ProgressUploadComplete, TimeSpan.FromSeconds(1)) + { + } + + public ProgressTracker(ProgressStatus progressStatus, Action progress, Action complete, TimeSpan progressInterval) + { + this.progressStatus = progressStatus; + this.progress = progress; + this.complete = complete; + this.progressInterval = progressInterval; + this.stopWatch = new Stopwatch(); + InitilizeProgressTimer(); + } + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "Lifetime of timer is bound to ProgressTracker")] + private void InitilizeProgressTimer() + { + stopWatch.Start(); + bool throwing = false; + try + { + // TODO: CLU + progressTimerOnElapsed = (state) => + { + ProgressRecord pr; + if (progressStatus.TryGetProgressRecord(out pr)) + { + this.progress(pr); + } + //progressTimer.Change(TimeSpan.Zero, TimeSpan.FromMilliseconds(progressInterval.TotalMilliseconds)); + }; + + progressTimer = new Timer(progressTimerOnElapsed, null, TimeSpan.Zero, TimeSpan.FromMilliseconds(progressInterval.TotalMilliseconds)); + /* + progressTimer = new Timer + { + AutoReset = false, + Interval = progressInterval.TotalMilliseconds + }; + progressTimerOnElapsed = (sender, args) => + { + ProgressRecord pr; + if (progressStatus.TryGetProgressRecord(out pr)) + { + this.progress(pr); + } + progressTimer.Enabled = true; + }; + progressTimer.Elapsed += progressTimerOnElapsed; + progressTimer.Enabled = true; + */ + } + catch (Exception) + { + throwing = true; + throw; + } + finally + { + if(throwing && progressTimer != null) + { + // TODO: CLU + //progressTimer.Change(Timeout.Infinite, Timeout.Infinite); + /* + progressTimer.Elapsed -= progressTimerOnElapsed; + progressTimer.Enabled = false; + */ + progressTimer.Dispose(); + progressTimer = null; + } + } + } + + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (isDisposed) + { + return; + } + if(disposing) + { + // TODO: CLU + //progressTimer.Change(Timeout.Infinite, Timeout.Infinite); + /* + progressTimer.Elapsed -= progressTimerOnElapsed; + progressTimer.Enabled = false; + */ + stopWatch.Stop(); + if (stopWatch.Elapsed != TimeSpan.Zero) + { + this.complete(stopWatch.Elapsed); + } + progressTimer.Dispose(); + this.isDisposed = true; + } + } + + } +} + \ No newline at end of file diff --git a/src/CLU/Sync/Properties/AssemblyInfo.cs b/src/CLU/Sync/Properties/AssemblyInfo.cs new file mode 100644 index 000000000000..1afa18298987 --- /dev/null +++ b/src/CLU/Sync/Properties/AssemblyInfo.cs @@ -0,0 +1,37 @@ +using System; +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Sync")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("Microsoft")] +[assembly: AssemblyProduct("Sync")] +[assembly: AssemblyCopyright("Copyright © Microsoft")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("fc29ef65-ccc2-4d41-b1c7-6e0437e479da")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] +[assembly: CLSCompliant(false)] \ No newline at end of file diff --git a/src/CLU/Sync/ServicePointHandler.cs b/src/CLU/Sync/ServicePointHandler.cs new file mode 100644 index 000000000000..139ec3832ea3 --- /dev/null +++ b/src/CLU/Sync/ServicePointHandler.cs @@ -0,0 +1,71 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Net; +using System.Reflection; + +namespace Microsoft.WindowsAzure.Commands.Sync +{ + internal class ServicePointHandler : IDisposable + { + private bool disposed; + // TODO: CLU + /* + private ServicePoint servicePoint; + private int originalConnectionLimit; + */ + + public ServicePointHandler(Uri uri, int connectionLimit) + { + // TODO: CLU + /* + this.servicePoint = ServicePointManager.FindServicePoint(uri, GetWebProxy()); + this.originalConnectionLimit = servicePoint.ConnectionLimit; + servicePoint.ConnectionLimit = connectionLimit; + */ + } + + private static IWebProxy GetWebProxy() + { + // TODO: CLU + return null; + /* + Type webRequestType = typeof(WebRequest); + PropertyInfo propertyInfo = webRequestType.GetProperty("InternalDefaultWebProxy", BindingFlags.Static | BindingFlags.NonPublic); + return (IWebProxy) propertyInfo.GetValue(null, null); + */ + } + + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if(!disposed) + { + if(disposing) + { + // TODO: CLU + //this.servicePoint.ConnectionLimit = originalConnectionLimit; + } + disposed = true; + } + } + } +} + \ No newline at end of file diff --git a/src/CLU/Sync/Sync.xproj b/src/CLU/Sync/Sync.xproj new file mode 100644 index 000000000000..d0dee083231f --- /dev/null +++ b/src/CLU/Sync/Sync.xproj @@ -0,0 +1,21 @@ + + + + 14.0 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) + + + + 6edcb32a-8420-48fc-99ce-94bea12d2fd2 + Microsoft.WindowsAzure.Commands.Sync + ..\artifacts\obj\$(MSBuildProjectName) + ..\artifacts\bin\$(MSBuildProjectName)\ + + + 2.0 + + + True + + + \ No newline at end of file diff --git a/src/CLU/Sync/Threading/Parallel.cs b/src/CLU/Sync/Threading/Parallel.cs new file mode 100644 index 000000000000..7f5f9691f8f3 --- /dev/null +++ b/src/CLU/Sync/Threading/Parallel.cs @@ -0,0 +1,240 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Threading; + +namespace Microsoft.WindowsAzure.Commands.Sync.Threading +{ + internal class Parallel + { + public readonly static int MaxParallellism = Environment.ProcessorCount; + + public static LoopResult ForEach(IEnumerable source, Func argumentConstructor, Action body) + { + return ForEach(source, argumentConstructor, body, MaxParallellism); + } + + public static LoopResult ForEach(IEnumerable source, Func argumentConstructor, Action body, int parallelism) + { + var loopResult = new InternalLoopResult(); + int numProcs = parallelism; + int remainingWorkItems = numProcs; + using (var enumerator = source.GetEnumerator()) + { + using (var mre = new ManualResetEvent(false)) + { + // Create each of the work items. + for (int p = 0; p < numProcs; p++) + { + ThreadPool.QueueUserWorkItem(delegate + { + try + { + + TA argument = argumentConstructor(); + // Iterate until there's no more work. + while (true) + { + // Get the next item under a lock, + // then process that item. + T nextItem; + lock (enumerator) + { + if (!enumerator.MoveNext()) break; + nextItem = enumerator.Current; + } + body(nextItem, argument); + } + } + catch (Exception e) + { + loopResult.AddException(e); + } + if (Interlocked.Decrement(ref remainingWorkItems) == 0) + mre.Set(); + }); + } + // Wait for all threads to complete. + mre.WaitOne(); + loopResult.SetCompleted(); + } + } + return loopResult; + } + + public static LoopResult ForEach(IEnumerable source, Action body) + { + return ForEach(source, body, MaxParallellism); + } + + public static LoopResult ForEach(IEnumerable source, Action body, int parallelism) + { + var loopResult = new InternalLoopResult(); + int numProcs = parallelism; + int remainingWorkItems = numProcs; + using (var enumerator = source.GetEnumerator()) + { + using (var mre = new ManualResetEvent(false)) + { + // Create each of the work items. + for (int p = 0; p < numProcs; p++) + { + ThreadPool.QueueUserWorkItem(delegate + { + // Iterate until there's no more work. + try + { + while (true) + { + // Get the next item under a lock, + // then process that item. + T nextItem; + lock (enumerator) + { + if (!enumerator.MoveNext()) break; + nextItem = enumerator.Current; + } + body(nextItem); + } + } + catch (Exception e) + { + loopResult.AddException(e); + } + if (Interlocked.Decrement(ref remainingWorkItems) == 0) + mre.Set(); + }); + } + // Wait for all threads to complete. + mre.WaitOne(); + loopResult.SetCompleted(); + } + } + return loopResult; + } + + public static LoopResult ForEach(IEnumerable source, Func argumentConstructor, Action body, Action finalize, int parallelism) + { + var loopResult = new InternalLoopResult(); + int numProcs = parallelism; + int remainingWorkItems = numProcs; + + IList arguments = new List(); + using (var enumerator = source.GetEnumerator()) + { + using (var mre = new ManualResetEvent(false)) + { + // Create each of the work items. + for (int p = 0; p < numProcs; p++) + { + ThreadPool.QueueUserWorkItem(delegate + { + try + { + TA argument = argumentConstructor(); + lock (arguments) + { + arguments.Add(argument); + } + // Iterate until there's no more work. + while (true && !loopResult.IsExceptional) + { + // Get the next item under a lock, + // then process that item. + T nextItem; + lock (enumerator) + { + if (!enumerator.MoveNext()) break; + nextItem = enumerator.Current; + } + body(nextItem, argument); + } + } + catch (Exception e) + { + loopResult.AddException(e); + } + if (Interlocked.Decrement(ref remainingWorkItems) == 0) + mre.Set(); + }); + } + // Wait for all threads to complete. + mre.WaitOne(); + + foreach (var argument in arguments) + { + finalize(argument); + } + + loopResult.SetCompleted(); + } + } + return loopResult; + } + + private class InternalLoopResult : LoopResult + { + private IList exceptions; + private object lockObject = new object(); + + public InternalLoopResult() + { + this.exceptions = new List(); + } + + public override bool IsCompleted + { + get; protected set; + } + + public override IList Exceptions + { + get { return new List(exceptions); } + } + + public override bool IsExceptional + { + get + { + lock (lockObject) + { + return this.exceptions.Count > 0; + } + } + } + + public void SetCompleted() + { + this.IsCompleted = true; + } + + public void AddException(Exception exception) + { + lock (lockObject) + { + this.exceptions.Add(exception); + } + } + } + } + + public abstract class LoopResult + { + public abstract bool IsCompleted { get; protected set; } + public abstract IList Exceptions { get; } + public abstract bool IsExceptional { get; } + } +} \ No newline at end of file diff --git a/src/CLU/Sync/Upload/BlobCreator.cs b/src/CLU/Sync/Upload/BlobCreator.cs new file mode 100644 index 000000000000..cf137650bac8 --- /dev/null +++ b/src/CLU/Sync/Upload/BlobCreator.cs @@ -0,0 +1,49 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System.IO; +using Microsoft.WindowsAzure.Commands.Sync.Download; + +namespace Microsoft.WindowsAzure.Commands.Sync.Upload +{ + public class BlobCreator : BlobCreatorBase + { + public BlobCreator(FileInfo localVhd, BlobUri destination, ICloudPageBlobObjectFactory blobObjectFactory, bool overWrite) : + base(localVhd, destination, blobObjectFactory, overWrite) + { + } + + protected override void CreateRemoteBlobAndPopulateContext(UploadContext context) + { + CreateRemoteBlob(); + PopulateContextWithUploadableRanges(localVhd, context, false); + PopulateContextWithDataToUpload(localVhd, context); + } + + private void CreateRemoteBlob() + { + Program.SyncOutput.MessageCreatingNewPageBlob(OperationMetaData.FileMetaData.VhdSize); + + // TODO: CLU + //destinationBlob.Create(OperationMetaData.FileMetaData.VhdSize); + destinationBlob.CreateAsync(OperationMetaData.FileMetaData.VhdSize).Wait(); + + using (var bdms = new BlobMetaDataScope(destinationBlob)) + { + bdms.Current.SetUploadMetaData(OperationMetaData); + bdms.Complete(); + } + } + } +} \ No newline at end of file diff --git a/src/CLU/Sync/Upload/BlobCreatorBase.cs b/src/CLU/Sync/Upload/BlobCreatorBase.cs new file mode 100644 index 000000000000..aa13a00c4ec6 --- /dev/null +++ b/src/CLU/Sync/Upload/BlobCreatorBase.cs @@ -0,0 +1,414 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +// TODO: CLU +//using System.Security.Permissions; +using System.ServiceModel.Channels; +using System.Text; +using System.Threading; +using Microsoft.WindowsAzure.Commands.Sync.Download; +using Microsoft.WindowsAzure.Commands.Tools.Vhd; +using Microsoft.WindowsAzure.Commands.Tools.Vhd.Model; +using Microsoft.WindowsAzure.Storage.Auth; +using Microsoft.WindowsAzure.Storage.Blob; + +namespace Microsoft.WindowsAzure.Commands.Sync.Upload +{ + public interface ICloudPageBlobObjectFactory + { + CloudPageBlob Create(BlobUri destination); + bool CreateContainer(BlobUri destination); + BlobRequestOptions CreateRequestOptions(); + } + + public abstract class BlobCreatorBase + { + private const long OneTeraByte = 1024L * 1024L * 1024L * 1024L; + + protected FileInfo localVhd; + protected readonly ICloudPageBlobObjectFactory blobObjectFactory; + protected Uri destination; + protected BlobUri blobDestination; + protected string queryString; + protected StorageCredentials credentials; + protected CloudPageBlob destinationBlob; + protected BlobRequestOptions requestOptions; + protected bool overWrite; + private readonly TimeSpan delayBetweenRetries = TimeSpan.FromSeconds(10); + + private const int MegaByte = 1024 * 1024; + private const int PageSizeInBytes = 2 * MegaByte; + private const int MaxBufferSize = 20 * MegaByte; + + protected BlobCreatorBase(FileInfo localVhd, BlobUri blobDestination, ICloudPageBlobObjectFactory blobObjectFactory, bool overWrite) + { + this.localVhd = localVhd; + this.blobObjectFactory = blobObjectFactory; + this.destination = new Uri(blobDestination.BlobPath); + this.blobDestination = blobDestination; + this.overWrite = overWrite; + + this.destinationBlob = blobObjectFactory.Create(blobDestination); + this.requestOptions = this.blobObjectFactory.CreateRequestOptions(); + } + + private LocalMetaData operationMetaData; + + public LocalMetaData OperationMetaData + { + // TODO: CLU + //[PermissionSet(SecurityAction.Demand, Name = "FullTrust")] + get + { + if (this.operationMetaData == null) + { + this.operationMetaData = new LocalMetaData + { + FileMetaData = FileMetaData.Create(localVhd.FullName), + SystemInformation = SystemInformation.Create() + }; + } + return operationMetaData; + } + } + + public byte[] MD5HashOfLocalVhd + { + get { return OperationMetaData.FileMetaData.MD5Hash; } + } + + private static void AssertIfValidVhdSize(FileInfo fileInfo) + { + using(var stream = new VirtualDiskStream(fileInfo.FullName)) + { + if(stream.Length > OneTeraByte) + { + var lengthString = stream.Length.ToString("N0", CultureInfo.CurrentCulture); + var expectedLengthString = OneTeraByte.ToString("N0", CultureInfo.CurrentCulture); + string message = String.Format("VHD size is too large ('{0}'), maximum allowed size is '{1}'.", lengthString, expectedLengthString); + throw new InvalidOperationException(message); + } + } + } + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "Need to keep UploadContext till the end of upload")] + public UploadContext Create() + { + AssertIfValidhVhd(localVhd); + AssertIfValidVhdSize(localVhd); + + this.blobObjectFactory.CreateContainer(blobDestination); + + UploadContext context = null; + bool completed = false; + try + { + context = new UploadContext + { + DestinationBlob = destinationBlob, + SingleInstanceMutex = AcquireSingleInstanceMutex(destinationBlob.Uri) + }; + + if (overWrite) + { + // TODO: CLU + //destinationBlob.DeleteIfExists(DeleteSnapshotsOption.IncludeSnapshots, null, requestOptions); + destinationBlob.DeleteIfExistsAsync(DeleteSnapshotsOption.IncludeSnapshots, null, requestOptions, null).Wait(); + } + + if (destinationBlob.Exists(requestOptions)) + { + Program.SyncOutput.MessageResumingUpload(); + + if(destinationBlob.GetBlobMd5Hash(requestOptions) != null) + { + throw new InvalidOperationException( + "An image already exists in blob storage with this name. If you want to upload again, use the Overwrite option."); + } + var metaData = destinationBlob.GetUploadMetaData(); + + AssertMetaDataExists(metaData); + AssertMetaDataMatch(metaData, OperationMetaData); + + PopulateContextWithUploadableRanges(localVhd, context, true); + PopulateContextWithDataToUpload(localVhd, context); + } + else + { + CreateRemoteBlobAndPopulateContext(context); + } + context.Md5HashOfLocalVhd = MD5HashOfLocalVhd; + completed = true; + } + finally + { + if(!completed && context != null) + { + context.Dispose(); + } + } + return context; + } + + public static void AssertIfValidhVhd(FileInfo vhdFile) + { + var vhdValidationResults = VhdValidator.Validate(VhdValidationType.IsVhd, vhdFile.FullName); + if (vhdValidationResults.Count(r => r.ErrorCode != 0) != 0) + { + string message = String.Format("'{0}' is not a valid VHD file.", vhdFile.FullName); + throw new InvalidOperationException(message, vhdValidationResults[0].Error); + } + } + + protected abstract void CreateRemoteBlobAndPopulateContext(UploadContext context); + + protected static void PopulateContextWithUploadableRanges(FileInfo vhdFile, UploadContext context, bool resume) + { + using (var vds = new VirtualDiskStream(vhdFile.FullName)) + { + IEnumerable ranges = vds.Extents.Select(e => e.Range).ToArray(); + + // TODO: CLU + var bs = new MemoryStream(); + vds.CopyTo(bs); + //var bs = new BufferedStream(vds); + if (resume) + { + // TODO: CLU + var alreadyUploadedRanges = context.DestinationBlob.GetPageRangesAsync().Result.Select(pr => new IndexRange(pr.StartOffset, pr.EndOffset)); + //var alreadyUploadedRanges = context.DestinationBlob.GetPageRanges().Select(pr => new IndexRange(pr.StartOffset, pr.EndOffset)); + ranges = IndexRange.SubstractRanges(ranges, alreadyUploadedRanges); + context.AlreadyUploadedDataSize = alreadyUploadedRanges.Sum(ir => ir.Length); + } + var uploadableRanges = IndexRangeHelper.ChunkRangesBySize(ranges, PageSizeInBytes).ToArray(); + if(vds.DiskType == DiskType.Fixed) + { + var nonEmptyUploadableRanges = GetNonEmptyRanges(bs, uploadableRanges).ToArray(); + context.UploadableDataSize = nonEmptyUploadableRanges.Sum(r => r.Length); + context.UploadableRanges = nonEmptyUploadableRanges; + } + else + { + context.UploadableDataSize = uploadableRanges.Sum(r => r.Length); + context.UploadableRanges = uploadableRanges; + } + } + } + + protected static void PopulateContextWithDataToUpload(FileInfo vhdFile, UploadContext context) + { + context.UploadableDataWithRanges = GetDataWithRangesToUpload(vhdFile, context); + } + + protected static IEnumerable GetNonEmptyRanges(Stream stream, IEnumerable uploadableRanges) + { + Program.SyncOutput.MessageDetectingActualDataBlocks(); + var manager = BufferManager.CreateBufferManager(Int32.MaxValue, MaxBufferSize); + int totalRangeCount = uploadableRanges.Count(); + int processedRangeCount = 0; + foreach (var range in uploadableRanges) + { + var dataWithRange = new DataWithRange(manager) + { + Data = ReadBytes(stream, range, manager), + Range = range + }; + using(dataWithRange) + { + if(dataWithRange.IsAllZero()) + { + Program.SyncOutput.DebugEmptyBlockDetected(dataWithRange.Range); + } + else + { + yield return dataWithRange.Range; + } + } + Program.SyncOutput.ProgressEmptyBlockDetection(++processedRangeCount, totalRangeCount); + } + + Program.SyncOutput.MessageDetectingActualDataBlocksCompleted(); + yield break; + } + + protected static IEnumerable GetDataWithRangesToUpload(FileInfo vhdFile, UploadContext context) + { + var uploadableRanges = context.UploadableRanges; + var manager = BufferManager.CreateBufferManager(Int32.MaxValue, MaxBufferSize); + using (var vds = new VirtualDiskStream(vhdFile.FullName)) + { + foreach (var range in uploadableRanges) + { + var localRange = range; + yield return new DataWithRange(manager) + { + Data = ReadBytes(vds, localRange, manager), + Range = localRange + }; + } + } + yield break; + } + + private static byte[] ReadBytes(Stream stream, IndexRange rangeToRead, BufferManager manager) + { + stream.Seek(rangeToRead.StartIndex, SeekOrigin.Begin); + + var bufferSize = (int)rangeToRead.Length; + var buffer = manager.TakeBuffer(bufferSize); + + for (int bytesRead = stream.Read(buffer, 0, bufferSize); + bytesRead < bufferSize; + bytesRead += stream.Read(buffer, bytesRead, bufferSize - bytesRead)) + { + } + return buffer; + } + + + private static void AssertMetaDataExists(LocalMetaData blobMetaData) + { + if(blobMetaData == null) + { + throw new InvalidOperationException("There is no CsUpload metadata on the blob, so CsUpload cannot resume. Use the overwrite option."); + } + } + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "Need to keep Mutex open till the end of upload")] + private static Mutex AcquireSingleInstanceMutex(Uri destinationBlobUri) + { + string mutexName = GetMutexName(destinationBlobUri); + + bool throwing = true; + Mutex singleInstanceMutex = null; + try + { + singleInstanceMutex = new Mutex(false, @"Global\" + mutexName); + // TODO: CLU + if (!singleInstanceMutex.WaitOne(TimeSpan.FromSeconds(5))) + //if (!singleInstanceMutex.WaitOne(TimeSpan.FromSeconds(5), false)) + { + var message = String.Format("An upload is already in progress on this machine"); + throw new InvalidOperationException(message); + } + throwing = false; + return singleInstanceMutex; + } + finally + { + if(throwing && singleInstanceMutex != null) + { + singleInstanceMutex.ReleaseMutex(); + // TODO: CLU + singleInstanceMutex.Dispose(); + //singleInstanceMutex.Close(); + } + } + } + + private static string GetMutexName(Uri destinationBlobUri) + { + var invariant = destinationBlobUri.ToString().ToLowerInvariant(); + var bytes = Encoding.Unicode.GetBytes(invariant); + using(var md5 = MD5.Create()) + { + byte[] hash = md5.ComputeHash(bytes); + return Convert.ToBase64String(hash); + } + } + + private static void AssertMetaDataMatch(LocalMetaData blobMetaData, LocalMetaData localMetaData) + { + var systemInformation = blobMetaData.SystemInformation; + + // TODO: CLU + if (String.Compare(systemInformation.MachineName, Environment.GetEnvironmentVariable("MachineName"), StringComparison.OrdinalIgnoreCase) != 0) + //if (String.Compare(systemInformation.MachineName, Environment.MachineName, CultureInfo.InvariantCulture, CompareOptions.IgnoreCase) != 0) + { + var message = String.Format("An upload is already in progress on machine {0} with process id {1}", + systemInformation.MachineName, + systemInformation.CsUploadProcessId); + + throw new InvalidOperationException(message); + } + + var fileMetaDataMessages = CompareFileMetaData(blobMetaData.FileMetaData, localMetaData.FileMetaData); + + if (fileMetaDataMessages.Count > 0) + { + throw new InvalidOperationException(fileMetaDataMessages.Aggregate((r,n)=>r + Environment.NewLine + n)); + } + } + + private static List CompareFileMetaData(FileMetaData blobFileMetaData, FileMetaData localFileMetaData) + { + var fileMetaDataMessages = new List(); + if (blobFileMetaData.VhdSize != localFileMetaData.VhdSize) + { + var message = String.Format("Logical size of VHD file in blob storage ({0}) and local VHD file ({1}) does not match ", + blobFileMetaData.VhdSize, + localFileMetaData.VhdSize); + fileMetaDataMessages.Add(message); + } + + if (blobFileMetaData.Size != localFileMetaData.Size) + { + var message = String.Format("Size of VHD file in blob storage ({0}) and local VHD file ({1}) does not match ", + blobFileMetaData.Size, + localFileMetaData.Size); + fileMetaDataMessages.Add(message); + } + + if (!blobFileMetaData.MD5Hash.SequenceEqual(localFileMetaData.MD5Hash)) + { + var message = String.Format("MD5 hash of VHD file in blob storage ({0}) and local VHD file ({1}) does not match ", + blobFileMetaData.MD5Hash.ToString(","), + localFileMetaData.MD5Hash.ToString(",")); + fileMetaDataMessages.Add(message); + } + + + if (DateTime.Compare(blobFileMetaData.LastModifiedDateUtc, localFileMetaData.LastModifiedDateUtc) != 0) + { + var message = String.Format("Last modified date of VHD file in blob storage ({0}) and local VHD file ({1}) does not match ", + blobFileMetaData.LastModifiedDateUtc, + localFileMetaData.LastModifiedDateUtc); + fileMetaDataMessages.Add(message); + } + + if (blobFileMetaData.FileFullName != localFileMetaData.FileFullName) + { + var message = String.Format("Full name of VHD file in blob storage ({0}) and local VHD file ({1}) does not match ", + blobFileMetaData.FileFullName, + localFileMetaData.FileFullName); + fileMetaDataMessages.Add(message); + } + + if (blobFileMetaData.CreatedDateUtc != localFileMetaData.CreatedDateUtc) + { + var message = String.Format("Full name of VHD file in blob storage ({0}) and local VHD file ({1}) does not match ", + blobFileMetaData.CreatedDateUtc, + localFileMetaData.CreatedDateUtc); + fileMetaDataMessages.Add(message); + } + return fileMetaDataMessages; + } + } +} \ No newline at end of file diff --git a/src/CLU/Sync/Upload/BlobSynchronizer.cs b/src/CLU/Sync/Upload/BlobSynchronizer.cs new file mode 100644 index 000000000000..47606738a4c1 --- /dev/null +++ b/src/CLU/Sync/Upload/BlobSynchronizer.cs @@ -0,0 +1,107 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using Microsoft.WindowsAzure.Commands.Sync.Threading; +using Microsoft.WindowsAzure.Storage.Blob; + +namespace Microsoft.WindowsAzure.Commands.Sync.Upload +{ + public class BlobSynchronizer + { + private readonly UploadContext context; + private byte[] md5Hash; + private readonly IEnumerable dataWithRanges; + private readonly int maxParallelism; + private readonly long dataToUpload; + private readonly long alreadyUploadedData; + private CloudPageBlob blob; + + public BlobSynchronizer(UploadContext context, int maxParallelism) + { + this.context = context; + this.md5Hash = context.Md5HashOfLocalVhd; + this.dataWithRanges = context.UploadableDataWithRanges; + this.dataToUpload = context.UploadableDataSize; + this.alreadyUploadedData = context.AlreadyUploadedDataSize; + this.blob = context.DestinationBlob; + this.maxParallelism = maxParallelism; + } + + public bool Synchronize() + { + var uploadStatus = new ProgressStatus(alreadyUploadedData, alreadyUploadedData + dataToUpload, new ComputeStats()); + + using(new ServicePointHandler(blob.Uri, this.maxParallelism)) + using(new ProgressTracker(uploadStatus)) + { + var loopResult = Parallel.ForEach(dataWithRanges, + () => new CloudPageBlob(blob.Uri, blob.ServiceClient.Credentials), + (dwr, b) => + { + using (dwr) + { + var md5HashOfDataChunk = GetBase64EncodedMd5Hash(dwr.Data, (int)dwr.Range.Length); + using (var stream = new MemoryStream(dwr.Data, 0, (int)dwr.Range.Length)) + { + b.Properties.ContentMD5 = md5HashOfDataChunk; + // TODO: CLU + b.WritePagesAsync(stream, dwr.Range.StartIndex, md5HashOfDataChunk).Wait(); + //b.WritePages(stream, dwr.Range.StartIndex); + } + } + uploadStatus.AddToProcessedBytes((int) dwr.Range.Length); + }, this.maxParallelism); + if(loopResult.IsExceptional) + { + if (loopResult.Exceptions.Any()) + { + Program.SyncOutput.ErrorUploadFailedWithExceptions(loopResult.Exceptions); + + throw new AggregateException(loopResult.Exceptions); + } + } + else + { + using(var bdms = new BlobMetaDataScope(new CloudPageBlob(blob.Uri, blob.ServiceClient.Credentials))) + { + bdms.Current.SetBlobMd5Hash(md5Hash); + bdms.Current.CleanUpUploadMetaData(); + bdms.Complete(); + } + } + } + return true; + } + + private static string GetBase64EncodedMd5Hash(byte[] data, int length) + { + using (var hash = MD5.Create()) + { + // TODO: CLU + var bytes = hash.ComputeHash(data, 0, length); + /* + hash.TransformBlock(data, 0, length, null, 0); + hash.TransformFinalBlock(new byte[0], 0, 0); + var bytes = hash.Hash; + */ + return Convert.ToBase64String(bytes); + } + } + } +} \ No newline at end of file diff --git a/src/CLU/Sync/Upload/ComputeStats.cs b/src/CLU/Sync/Upload/ComputeStats.cs new file mode 100644 index 000000000000..f4369f876fc0 --- /dev/null +++ b/src/CLU/Sync/Upload/ComputeStats.cs @@ -0,0 +1,51 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System.Collections.Generic; + +namespace Microsoft.WindowsAzure.Commands.Tools.CsUpload +{ + internal class ComputeStats + { + IList history; + int historySize; + + public ComputeStats() : this(60) + { + } + + public ComputeStats(int historySize) + { + history = new List(historySize); + this.historySize = historySize; + } + + public double ComputeAvg(double current) + { + if (history.Count > historySize) + { + history.RemoveAt(0); + } + history.Add(current); + double sum = 0.0; + foreach (var x in history) + { + sum += x; + } + return sum / history.Count; + } + } +} + + diff --git a/src/CLU/Sync/Upload/DataWithRange.cs b/src/CLU/Sync/Upload/DataWithRange.cs new file mode 100644 index 000000000000..448b13e9bd37 --- /dev/null +++ b/src/CLU/Sync/Upload/DataWithRange.cs @@ -0,0 +1,62 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.ServiceModel.Channels; +using Microsoft.WindowsAzure.Commands.Tools.Vhd.Model; + +namespace Microsoft.WindowsAzure.Commands.Sync.Upload +{ + public class DataWithRange : IDisposable + { + private readonly BufferManager manager; + private bool disposed; + + public DataWithRange(BufferManager manager) + { + this.manager = manager; + } + + public bool IsAllZero() + { + var startIndex = Array.FindIndex(this.Data, 0, (int)this.Range.Length, b => b != 0); + + return startIndex == -1; + } + + public byte[] Data { get; set; } + public IndexRange Range { get; set; } + + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (this.disposed) + { + return; + } + + if (disposing) + { + this.manager.ReturnBuffer(this.Data); + this.Data = null; + } + this.disposed = true; + } + } +} \ No newline at end of file diff --git a/src/CLU/Sync/Upload/ExtensionMethods.cs b/src/CLU/Sync/Upload/ExtensionMethods.cs new file mode 100644 index 000000000000..43c77421a7ba --- /dev/null +++ b/src/CLU/Sync/Upload/ExtensionMethods.cs @@ -0,0 +1,234 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Microsoft.WindowsAzure.Commands.Tools.Vhd.Model; +using Microsoft.WindowsAzure.Commands.Tools.Vhd.Model.Persistence; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; + +namespace Microsoft.WindowsAzure.Commands.Sync.Upload +{ + internal static class CloudPageBlobExtensions + { + public static void SetUploadMetaData(this CloudPageBlob blob, LocalMetaData metaData) + { + if (metaData == null) + { + throw new ArgumentNullException("metaData"); + } + blob.Metadata[LocalMetaData.MetaDataKey] = SerializationUtil.GetSerializedString(metaData); + } + + public static void CleanUpUploadMetaData(this CloudPageBlob blob) + { + blob.Metadata.Remove(LocalMetaData.MetaDataKey); + } + + public static LocalMetaData GetUploadMetaData(this CloudPageBlob blob) + { + if (blob.Metadata.Keys.Contains(LocalMetaData.MetaDataKey)) + { + return SerializationUtil.GetObjectFromSerializedString(blob.Metadata[LocalMetaData.MetaDataKey]); + } + return null; + } + + public static byte[] GetBlobMd5Hash(this CloudPageBlob blob) + { + // TODO: CLU + blob.FetchAttributesAsync().Wait(); + //blob.FetchAttributes(); + if (String.IsNullOrEmpty(blob.Properties.ContentMD5)) + { + return null; + } + + return Convert.FromBase64String(blob.Properties.ContentMD5); + } + + public static byte[] GetBlobMd5Hash(this CloudPageBlob blob, BlobRequestOptions requestOptions) + { + // TODO: CLU + blob.FetchAttributesAsync(new AccessCondition(), requestOptions, null).Wait(); + //blob.FetchAttributes(new AccessCondition(), requestOptions); + if (String.IsNullOrEmpty(blob.Properties.ContentMD5)) + { + return null; + } + + return Convert.FromBase64String(blob.Properties.ContentMD5); + } + + public static void SetBlobMd5Hash(this CloudPageBlob blob, byte[] md5Hash) + { + var base64String = Convert.ToBase64String(md5Hash); + blob.Properties.ContentMD5 = base64String; + } + + public static void RemoveBlobMd5Hash(this CloudPageBlob blob) + { + blob.Properties.ContentMD5 = null; + } + + public static VhdFooter GetVhdFooter(this CloudPageBlob basePageBlob) + { + var vhdFileFactory = new VhdFileFactory(); + // TODO: CLU + using (var file = vhdFileFactory.Create(basePageBlob.OpenReadAsync().Result)) + //using (var file = vhdFileFactory.Create(basePageBlob.OpenRead())) + { + return file.Footer; + } + } + + public static bool Exists(this CloudPageBlob blob) + { + // TODO: CLU + var listBlobItems = blob.Container.ListBlobsSegmentedAsync(null).Result.Results; + //var listBlobItems = blob.Container.ListBlobs(); + var blobToUpload = listBlobItems.FirstOrDefault(b => b.Uri == blob.Uri); + if(blobToUpload is CloudBlockBlob) + { + var message = String.Format(" CsUpload is expecting a page blob, however a block blob was found: '{0}'.", blob.Uri); + throw new InvalidOperationException(message); + } + return blobToUpload != null; + } + + public static bool Exists(this CloudPageBlob blob, BlobRequestOptions options) + { + // TODO: CLU + var listBlobItems = blob.Container.ListBlobsSegmentedAsync(null, false, BlobListingDetails.UncommittedBlobs, null, null, options, null).Result.Results; + //var listBlobItems = blob.Container.ListBlobs(null,false, BlobListingDetails.UncommittedBlobs, options); + var blobToUpload = listBlobItems.FirstOrDefault(b => b.Uri == blob.Uri); + if (blobToUpload is CloudBlockBlob) + { + var message = String.Format(" CsUpload is expecting a page blob, however a block blob was found: '{0}'.", blob.Uri); + throw new InvalidOperationException(message); + } + return blobToUpload != null; + } + } + + internal static class StringExtensions + { + public static string ToString(this IEnumerable source, string separator) + { + return "[" + string.Join(",", source.Select(s=>s.ToString()).ToArray()) + "]"; + } + } + + public class VhdFilePath + { + public VhdFilePath(string absolutePath, string relativePath) + { + AbsolutePath = absolutePath; + RelativePath = relativePath; + } + + public string AbsolutePath { get; private set; } + public string RelativePath { get; private set; } + } + + internal static class VhdFileExtensions + { + public static IEnumerable GetChildrenIds(this VhdFile vhdFile, Guid uniqueId) + { + var identityChain = vhdFile.GetIdentityChain(); + + if (!identityChain.Contains(uniqueId)) + { + yield break; + } + + foreach (var id in identityChain.TakeWhile(id => id != uniqueId)) + { + yield return id; + } + } + + public static VhdFilePath GetFilePathBy(this VhdFile vhdFile, Guid uniqueId) + { + VhdFilePath result = null; + string baseVhdPath = String.Empty; + var newBlocksOwners = new List { Guid.Empty }; + + var current = vhdFile; + while (current != null && current.Footer.UniqueId != uniqueId) + { + newBlocksOwners.Add(current.Footer.UniqueId); + if (current.Parent != null) + { + result = new VhdFilePath(current.Header.GetAbsoluteParentPath(), + current.Header.GetRelativeParentPath()); + } + current = current.Parent; + } + + if (result == null) + { + string message = String.Format("There is no parent VHD file with with the id '{0}'", uniqueId); + throw new InvalidOperationException(message); + } + + return result; + } + } + + public static class UriExtensions + { + /// + /// Normalizes a URI for use as a blob URI. + /// + /// + /// Ensures that the container name is lower-case. + /// + public static Uri NormalizeBlobUri(this Uri uri) + { + var ub = new UriBuilder(uri); + var parts = ub.Path + .Split(new char[] { '/' }, StringSplitOptions.None) + .Select((p, i) => i == 1 ? p.ToLowerInvariant() : p) + .ToArray(); + ub.Path = string.Join("/", parts); + return ub.Uri; + } + + } + + public static class ExceptionUtil + { + public static string DumpStorageExceptionErrorDetails(StorageException storageException) + { + if(storageException == null) + { + return string.Empty; + } + + var message = new StringBuilder(); + message.AppendLine("StorageException details"); + message.Append("Error.Code:").AppendLine(storageException.RequestInformation.ExtendedErrorInformation.ErrorCode); + message.Append("ErrorMessage:").AppendLine(storageException.RequestInformation.ExtendedErrorInformation.ErrorMessage); + foreach (var key in storageException.RequestInformation.ExtendedErrorInformation.AdditionalDetails.Keys) + { + message.Append(key).Append(":").Append(storageException.RequestInformation.ExtendedErrorInformation.AdditionalDetails[key]); + } + return message.ToString(); + } + } +} diff --git a/src/CLU/Sync/Upload/IndexRangeHelper.cs b/src/CLU/Sync/Upload/IndexRangeHelper.cs new file mode 100644 index 000000000000..af02b700947a --- /dev/null +++ b/src/CLU/Sync/Upload/IndexRangeHelper.cs @@ -0,0 +1,60 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.Linq; +using Microsoft.WindowsAzure.Commands.Tools.Vhd.Model; + +namespace Microsoft.WindowsAzure.Commands.Sync.Upload +{ + internal static class IndexRangeHelper + { + public static IEnumerable ChunkRangesBySize(IEnumerable extents, int pageSizeInBytes) + { + var extentRanges = extents.SelectMany(e => e.PartitionBy(pageSizeInBytes)); + var extentQueue = new Queue(extentRanges); + + if (extentQueue.Count == 0) + { + yield break; + } + + // move to next start position + do + { + var result = extentQueue.Dequeue(); + while (result.Length < pageSizeInBytes && extentQueue.Count > 0) + { + var nextRange = extentQueue.Peek(); + if (!nextRange.Abuts(result)) + { + break; + } + var mergedRange = nextRange.Merge(result); + if (mergedRange.Length <= pageSizeInBytes) + { + result = mergedRange; + extentQueue.Dequeue(); + } + else + { + break; + } + } + yield return result; + } while (extentQueue.Count > 0); + } + + } +} \ No newline at end of file diff --git a/src/CLU/Sync/Upload/PatchingBlobCreator.cs b/src/CLU/Sync/Upload/PatchingBlobCreator.cs new file mode 100644 index 000000000000..ac41097a13e6 --- /dev/null +++ b/src/CLU/Sync/Upload/PatchingBlobCreator.cs @@ -0,0 +1,228 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using Microsoft.WindowsAzure.Commands.Sync.Download; +using Microsoft.WindowsAzure.Commands.Tools.Vhd; +using Microsoft.WindowsAzure.Commands.Tools.Vhd.Model.Persistence; +using Microsoft.WindowsAzure.Storage.Blob; + +namespace Microsoft.WindowsAzure.Commands.Sync.Upload +{ + public class PatchingBlobCreator : BlobCreatorBase + { + protected Uri baseVhdBlob; + protected BlobUri baseVhdBlobUri; + + public PatchingBlobCreator(FileInfo localVhd, BlobUri destination, BlobUri baseVhdBlob, ICloudPageBlobObjectFactory blobObjectFactory, bool overWrite) : + base(localVhd, destination, blobObjectFactory, overWrite) + { + this.baseVhdBlob = baseVhdBlob.Uri; + this.baseVhdBlobUri = baseVhdBlob; + } + + + protected override void CreateRemoteBlobAndPopulateContext(UploadContext context) + { + CreateRemoteBlob(); + PopulateContextWithUploadableRanges(localVhd, context, true); + PopulateContextWithDataToUpload(localVhd, context); + } + + private void CreateRemoteBlob() + { + var baseBlob = this.blobObjectFactory.Create(baseVhdBlobUri); + + if(!baseBlob.Exists()) + { + throw new InvalidOperationException(String.Format("Base image to patch doesn't exist in blob storage: {0}", baseVhdBlobUri.Uri)); + } + var blobVhdFooter = baseBlob.GetVhdFooter(); + + long blobSize; + VhdFilePath localBaseVhdPath; + IEnumerable childrenVhdIds; + using (var vhdFile = new VhdFileFactory().Create(localVhd.FullName)) + { + localBaseVhdPath = vhdFile.GetFilePathBy(blobVhdFooter.UniqueId); + childrenVhdIds = vhdFile.GetChildrenIds(blobVhdFooter.UniqueId).ToArray(); + blobSize = vhdFile.Footer.VirtualSize; + } + + FileMetaData fileMetaData = GetFileMetaData(baseBlob, localBaseVhdPath); + + var md5Hash = baseBlob.GetBlobMd5Hash(); + if (!md5Hash.SequenceEqual(fileMetaData.MD5Hash)) + { + var message = String.Format("Patching cannot proceed, MD5 hash of base image in blob storage ({0}) and base VHD file ({1}) does not match ", + baseBlob.Uri, + localBaseVhdPath); + throw new InvalidOperationException(message); + } + + Program.SyncOutput.MessageCreatingNewPageBlob(blobSize); + + CopyBaseImageToDestination(); + + using (var vds = new VirtualDiskStream(localVhd.FullName)) + { + var streamExtents = vds.Extents.ToArray(); + var enumerable = streamExtents.Where(e => childrenVhdIds.Contains(e.Owner)).ToArray(); + foreach (var streamExtent in enumerable) + { + var indexRange = streamExtent.Range; + // TODO: CLU + destinationBlob.ClearPagesAsync(indexRange.StartIndex, indexRange.Length).Wait(); + //destinationBlob.ClearPages(indexRange.StartIndex, indexRange.Length); + } + } + + using(var bmds = new BlobMetaDataScope(destinationBlob)) + { + bmds.Current.RemoveBlobMd5Hash(); + bmds.Current.SetUploadMetaData(OperationMetaData); + bmds.Complete(); + } + } + + private void CopyBaseImageToDestination() + { + var source = this.blobObjectFactory.Create(baseVhdBlobUri); + // TODO: CLU + source.FetchAttributesAsync().Wait(); + //source.FetchAttributes(); + + var copyStatus = new ProgressStatus(0, source.Properties.Length); + using (new ProgressTracker(copyStatus, Program.SyncOutput.ProgressCopyStatus, Program.SyncOutput.ProgressCopyComplete,TimeSpan.FromSeconds(1))) + { + // TODO: CLU + destinationBlob.StartCopyAsync(source).Wait(); + //destinationBlob.StartCopy(source); + // TODO: CLU + destinationBlob.FetchAttributesAsync().Wait(); + //destinationBlob.FetchAttributes(); + + while (true) + { + if (destinationBlob.CopyState.BytesCopied != null) + { + copyStatus.AddToProcessedBytes(destinationBlob.CopyState.BytesCopied.Value - copyStatus.BytesProcessed); + } + if (destinationBlob.CopyState.Status == CopyStatus.Success) + { + break; + } + if (destinationBlob.CopyState.Status == CopyStatus.Pending) + { + Thread.Sleep(TimeSpan.FromSeconds(1)); + } + else + { + // TODO: CLU + throw new Exception( + string.Format("Cannot copy source '{0}' to destination '{1}', copy state is '{2}'", source.Uri, + destinationBlob.Uri, destinationBlob.CopyState)); + /* + throw new ApplicationException( + string.Format("Cannot copy source '{0}' to destination '{1}', copy state is '{2}'", source.Uri, + destinationBlob.Uri, destinationBlob.CopyState)); + */ + } + // TODO: CLU + destinationBlob.FetchAttributesAsync().Wait(); + //destinationBlob.FetchAttributes(); + } + } + } + + private FileMetaData GetFileMetaData(CloudPageBlob baseBlob, VhdFilePath localBaseVhdPath) + { + FileMetaData fileMetaData; + if(File.Exists(localBaseVhdPath.AbsolutePath)) + { + fileMetaData = FileMetaData.Create(localBaseVhdPath.AbsolutePath); + } + else + { + var filePath = Path.Combine(localVhd.Directory.FullName, localBaseVhdPath.RelativePath); + if (File.Exists(filePath)) + { + fileMetaData = FileMetaData.Create(filePath); + } + else + { + var message = String.Format("Cannot find the local base image for '{0}' in neither of the locations '{1}', '{2}'.", + baseBlob.Uri, + localBaseVhdPath.AbsolutePath, + localBaseVhdPath.RelativePath); + throw new InvalidOperationException(message); + } + } + return fileMetaData; + } + } + + class BlobMetaDataScope : IDisposable + { + private readonly CloudPageBlob blob; + private bool disposed; + private bool completed; + + public BlobMetaDataScope(CloudPageBlob blob) + { + this.blob = blob; + // TODO: CLU + this.blob.FetchAttributesAsync().Wait(); + //this.blob.FetchAttributes(); + this.Current = blob; + } + + public CloudPageBlob Current { get; private set; } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + protected void Dispose(bool disposing) + { + if(disposing) + { + if(!this.disposed) + { + if(completed) + { + // TODO: CLU + this.blob.SetMetadataAsync().Wait(); + //this.blob.SetMetadata(); + // TODO: CLU + this.blob.SetPropertiesAsync().Wait(); + //this.blob.SetProperties(); + } + this.disposed = true; + } + } + } + + public void Complete() + { + this.completed = true; + } + } +} \ No newline at end of file diff --git a/src/CLU/Sync/Upload/SerializationUtil.cs b/src/CLU/Sync/Upload/SerializationUtil.cs new file mode 100644 index 000000000000..0964fdcc02ae --- /dev/null +++ b/src/CLU/Sync/Upload/SerializationUtil.cs @@ -0,0 +1,46 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System.IO; +using System.Runtime.Serialization; +using System.Text; + +namespace Microsoft.WindowsAzure.Commands.Sync.Upload +{ + internal class SerializationUtil + { + public static T GetObjectFromSerializedString(string data) + { + var serializer = new DataContractSerializer(typeof(T)); + using (var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes(data))) + { + return (T)serializer.ReadObject(memoryStream); + } + } + + public static string GetSerializedString(T localMetaData) + { + var serializer = new DataContractSerializer(typeof(T)); + string serializedString; + using (var memoryStream = new MemoryStream()) + { + serializer.WriteObject(memoryStream, localMetaData); + memoryStream.Flush(); + byte[] bytes = memoryStream.ToArray(); + serializedString = Encoding.UTF8.GetString(bytes, 0, bytes.Length); + } + return serializedString; + } + } +} \ No newline at end of file diff --git a/src/CLU/Sync/Upload/UploadContext.cs b/src/CLU/Sync/Upload/UploadContext.cs new file mode 100644 index 000000000000..75405fb96dcd --- /dev/null +++ b/src/CLU/Sync/Upload/UploadContext.cs @@ -0,0 +1,68 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Threading; +using Microsoft.WindowsAzure.Commands.Tools.Vhd.Model; +using Microsoft.WindowsAzure.Storage.Blob; + +namespace Microsoft.WindowsAzure.Commands.Sync.Upload +{ + public class UploadContext : IDisposable + { + private bool disposed; + + public CloudPageBlob DestinationBlob { get; set; } + + public IEnumerable UploadableDataWithRanges { get; set; } + + public IEnumerable UploadableRanges { get; set; } + + public long UploadableDataSize { get; set; } + + public long AlreadyUploadedDataSize { get; set; } + + public byte[] Md5HashOfLocalVhd { get; set; } + + public Mutex SingleInstanceMutex { get; set; } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (disposed) + { + return; + } + + if(disposing) + { + if (SingleInstanceMutex != null) + { + SingleInstanceMutex.ReleaseMutex(); + // TODO: CLU + SingleInstanceMutex.Dispose(); + //SingleInstanceMutex.Close(); + } + + disposed = true; + } + } + } +} \ No newline at end of file diff --git a/src/CLU/Sync/Upload/UploadOperationMetaData.cs b/src/CLU/Sync/Upload/UploadOperationMetaData.cs new file mode 100644 index 000000000000..8c21363394e1 --- /dev/null +++ b/src/CLU/Sync/Upload/UploadOperationMetaData.cs @@ -0,0 +1,166 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + + +using System; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Runtime.Serialization; +using System.Security.Cryptography; +// TODO: CLU +//using System.Security.Permissions; +using Microsoft.WindowsAzure.Commands.Sync.IO; +using Microsoft.WindowsAzure.Commands.Tools.Vhd; + +namespace Microsoft.WindowsAzure.Commands.Sync.Upload +{ + [DataContract] + public class FileMetaData + { + [DataMember(EmitDefaultValue = false, Order = 10)] + public string FileFullName { get; set; } + + [DataMember(EmitDefaultValue = false, Order = 20)] + private string InternalCreatedDateUtc { get; set; } + + [DataMember(EmitDefaultValue = false, Order = 30)] + private string InternalLastModifiedDateUtc { get; set; } + + [DataMember(EmitDefaultValue = false, Order = 40)] + public long Size { get; set; } + + [DataMember(EmitDefaultValue = false, Order = 50)] + public long VhdSize { get; set; } + + [DataMember(EmitDefaultValue = false, Order = 60)] + public byte[] MD5Hash { get; set; } + + public DateTime CreatedDateUtc + { + get { return DateTime.Parse(this.InternalCreatedDateUtc, DateTimeFormatInfo.InvariantInfo); } + set { this.InternalCreatedDateUtc = value.ToString(DateTimeFormatInfo.InvariantInfo); } + } + + public DateTime LastModifiedDateUtc + { + get { return DateTime.Parse(this.InternalLastModifiedDateUtc, DateTimeFormatInfo.InvariantInfo); } + set { this.InternalLastModifiedDateUtc = value.ToString(DateTimeFormatInfo.InvariantInfo); } + } + + public static FileMetaData Create(string filePath) + { + var fileInfo = new FileInfo(filePath); + if(!fileInfo.Exists) + { + throw new FileNotFoundException(filePath); + } + + using(var stream = new VirtualDiskStream(filePath)) + { + return new FileMetaData + { + FileFullName = fileInfo.FullName, + CreatedDateUtc = DateTime.SpecifyKind(fileInfo.CreationTimeUtc, DateTimeKind.Utc), + LastModifiedDateUtc = DateTime.SpecifyKind(fileInfo.LastWriteTimeUtc, DateTimeKind.Utc), + Size = fileInfo.Length, + VhdSize = stream.Length, + MD5Hash = CalculateMd5Hash(stream, filePath) + }; + } + } + + private static byte[] CalculateMd5Hash(Stream stream, string filePath) + { + using(var md5 = MD5.Create()) + { + using (var swrp = new StreamWithReadProgress(stream, TimeSpan.FromSeconds(1))) + { + // TODO: CLU + var bs = new MemoryStream(); + swrp.CopyTo(bs); + swrp.Seek(0, SeekOrigin.Begin); + //var bs = new BufferedStream(swrp); + Program.SyncOutput.MessageCalculatingMD5Hash(filePath); + var md5Hash = md5.ComputeHash(bs); + Program.SyncOutput.MessageMD5HashCalculationFinished(); + return md5Hash; + } + } + } + } + + [DataContract] + public class SystemInformation + { + [DataMember(EmitDefaultValue = false, Order = 10)] + public string MachineName { get; set; } + + [DataMember(EmitDefaultValue = false, Order = 20)] + public int CsUploadProcessId { get; set; } + + // TODO: CLU + //[PermissionSet(SecurityAction.LinkDemand, Name = "FullTrust")] + public static SystemInformation Create() + { + return new SystemInformation + { + // TODO: CLU + MachineName = Environment.GetEnvironmentVariable("%COMPUTERNAME%"), + //MachineName = Environment.MachineName, + CsUploadProcessId = Process.GetCurrentProcess().Id + }; + } + } + + + [DataContract] + public class LocalMetaData + { + public static string MetaDataKey = "localmetadata"; + + [DataMember(EmitDefaultValue = false, Order = 10)] + public FileMetaData FileMetaData { get; set; } + + [DataMember(EmitDefaultValue = false, Order = 20)] + public SystemInformation SystemInformation { get; set; } + } + + [DataContract] + public class LeaseMetaData + { + public static string MetaDataKey = "leasemetadata"; + + [DataMember(EmitDefaultValue = false, Order = 10)] + public Guid LeaseId { get; set; } + + [DataMember(EmitDefaultValue = false, Order = 20)] + public DateTime LeaseExpirationDateUtc { get; set; } + } + + [DataContract] + public class BlobMetaData + { + public static string MetaDataKey = "blobmetadata"; + + [DataMember(EmitDefaultValue = false, Order = 10)] + public string ETag { get; set; } + + [DataMember(EmitDefaultValue = false, Order = 20)] + public DateTime LastModifiedUtc { get; set; } + + [DataMember(EmitDefaultValue = false, Order = 30)] + public byte[] MD5Hash { get; set; } + } +} \ No newline at end of file diff --git a/src/CLU/Sync/project.json b/src/CLU/Sync/project.json new file mode 100644 index 000000000000..4fdcca933356 --- /dev/null +++ b/src/CLU/Sync/project.json @@ -0,0 +1,58 @@ +{ + "version": "1.0.0-*", + "description": "Microsoft.WindowsAzure.Commands.Tools.Vhd", + "authors": [ "huangpf, markcowl, hovsepm, haocs" ], + "tags": [ "" ], + "projectUrl": "", + "licenseUrl": "", + "frameworks": { + "dnxcore50": { + "dependencies": { + "Microsoft.NETCore": "5.0.1-beta-23516", + "Microsoft.NETCore.Platforms": "1.0.1-beta-23516", + "Microsoft.CSharp": "4.0.1-beta-23516" + } + } + }, + "dependencies": { + "System.Linq": "4.0.1-beta-23516", + "Microsoft.CLU": "1.0.0", + "Commands.Common": "", + "Commands.Common.Authentication": "", + "Commands.Common.Storage": "", + "Commands.ResourceManager.Common": "", + "VhdManagement": "", + "Microsoft.IdentityModel.Clients.ActiveDirectory": "3.6.212041202-alpha", + "Microsoft.Rest.ClientRuntime": "1.8.1", + "Newtonsoft.Json": "7.0.1", + "System.Collections": "4.0.11-beta-23516", + "System.Collections.Concurrent": "4.0.11-beta-23516", + "System.Diagnostics.TraceSource": "4.0.0-beta-23516", + "System.Diagnostics.Tracing": "4.0.21-beta-23516", + "System.IO": "4.0.11-beta-23516", + "System.IO.FileSystem": "4.0.1-beta-23516", + "System.Net.WebHeaderCollection": "4.0.1-beta-23516", + "System.Reflection": "4.1.0-beta-23516", + "System.Reflection.Extensions": "4.0.1-beta-23516", + "System.Reflection.Primitives": "4.0.1-beta-23516", + "System.Reflection.TypeExtensions": "4.1.0-beta-23516", + "System.Runtime": "4.0.21-beta-23516", + "System.Runtime.Extensions": "4.0.11-beta-23516", + "System.Runtime.Serialization.Primitives": "4.1.0-beta-23516", + "System.Runtime.Serialization.Json": "4.0.1-beta-23516", + "System.Runtime.Serialization.Xml": "4.1.0-beta-23516", + "System.Security.Cryptography.Algorithms": "4.0.0-beta-23516", + "System.Security.Cryptography.X509Certificates": "4.0.0-beta-23516", + "System.Text.Encoding": "4.0.11-beta-23516", + "System.Text.Encoding.Extensions": "4.0.11-beta-23516", + "System.Threading": "4.0.11-beta-23516", + "System.Threading.Tasks": "4.0.11-beta-23516", + "System.Threading.Thread": "4.0.0-beta-23516", + "System.Threading.Timer": "4.0.1-beta-23516", + "System.Xml.XmlDocument": "4.0.1-beta-23516", + "System.Xml.XPath.XmlDocument": "4.0.1-beta-23516", + "WindowsAzure.Storage": "6.1.1-preview", + "System.ServiceModel.Primitives": "4.1.0-beta-23516" + }, + "compilationOptions": { "emitEntryPoint": true } +} diff --git a/src/CLU/VhdManagement/Async/AsyncMachine.cs b/src/CLU/VhdManagement/Async/AsyncMachine.cs new file mode 100644 index 000000000000..0e65310cb856 --- /dev/null +++ b/src/CLU/VhdManagement/Async/AsyncMachine.cs @@ -0,0 +1,1150 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Threading; + +namespace Microsoft.WindowsAzure.Commands.Tools.Common.General +{ + // + // The asynchronous machine below supports two modes, either all operations are required to complete or + // only a subset of them (quorum) is required to proceed. There is also a completion port optimization + // for a single asynchronous operation (CompletionPort.SingleOperation) to avoid unnecessary memory allocations. + // + // Examples: + // + // Scenario 1: Single asynchronous operation without an explicit timeout + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + // + // ... + // AsyncMachine machine = new AsyncMachine(); + // machine.Start(this.ProcessMessage(asyncMachine, message)); + // ... + // + // IEnumerable ProcessMessage(AsyncMachine machine, Message message) + // { + // ... + // WebRequest request = WebRequest.Create(); + // + // request.BeginGetResponse(machine.CompletionCallback, null); + // + // yield return CompletionPort.SingleOperation; + // + // ProcessWebResponse(request, machine.CompletionResult); + // ... + // + // + // Scenario 2: Multiple asynchronous operations with time out + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + // + // IEnumerable ProcessMessage(AsyncMachine machine, Message message) + // { + // ... + // CompletionPort port = machine.CreateCompletionPort(TimeSpan.FromSeconds(10)); + // + // foreach (WebRequest request in pendingRequests) + // { + // request.BeginGetResponse(port[request].CompletionCallback, null); + // } + // + // yield return port; + // + // foreach (WebRequest request in pendingRequests) + // { + // // The operation is not completed if it's timed out + // if (port[request].IsCompleted) + // { + // ProcessWebResponse(request, port[request].CompletionResult); + // } + // else + // { + // port[request].Cancel(request.EndGetResponse); + // } + // } + // ... + // + // + // Scenario 3: Multiple asynchronous operations with quorum and time out + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + // + // IEnumerable ProcessMessage(AsyncMachine machine, Message message) + // { + // ... + // + // // The async machine will resume if 3 operations out of 5 will complete + // CompletionPort port = machine.CreateCompletionPort(5, 3, TimeSpan.FromSeconds(10)); + // + // for (int c = 0; c < 5; ++c) + // { + // WebRequest request = pendingRequests[c]; + // + // request.BeginGetResponse(port[request].CompletionCallback, null); + // } + // + // yield return port; + // + // for (int c = 0; c < 5; ++c) + // { + // WebRequest request = pendingRequests[c]; + // + // // The operation is not completed if it's timed out + // if (port[request].IsCompleted) + // { + // ProcessWebResponse(request, port[request].CompletionResult); + // } + // else + // { + // port[request].Cancel(request.EndGetResponse); + // } + // } + // ... + // + + // Represents an asynchronous operation + public class AsyncOperation + { + #region Constructors + + internal AsyncOperation(Action checkOperationCompletion) + { + Debug.Assert(checkOperationCompletion != null); + + this.CheckOperationCompletion = checkOperationCompletion; + this.CompletionCallback = this.AsyncCallback; + } + + #endregion + + public AsyncCallback CompletionCallback { get; private set; } + + public IAsyncResult CompletionResult + { + get + { + if (this.completionResult == null) + { + throw new InvalidOperationException("Completion result is not available yet."); + } + + return this.completionResult; + } + } + + // Indicates whether a particular asynchronous operation is completed + public bool IsCompleted { get; internal set; } + + // Cancels pending async operation meaning that we'll call the corresponding end operation whenever it completes + // (since there is no currently real cancelation mechanism for .NET async operations) + public void Cancel(AsyncCallback endAsyncOperation) + { + if (endAsyncOperation == null) + { + throw new ArgumentNullException("endAsyncOperation"); + } + + this.EndAsynchronousOperation = endAsyncOperation; + + if (this.CompletionResult != null && Interlocked.CompareExchange(ref this.isEndOperationCalled, True, False) == False) + { + // The operation is completed but end async operation is not called yet, need to call it + this.EndAsynchronousOperation(this.CompletionResult); + } + } + + #region Private Members + + // Gets invoked when an asynchronous operation has completed + private void AsyncCallback(IAsyncResult result) + { + if (result == null) + { + throw new ArgumentNullException("result"); + } + + this.completionResult = result; + + // this.CheckOperationCompletion will set AsyncOperation.IsCompleted to false if quorum is reached or timeout is expired, + // and async machine has resumed its execution. In this case we'll consider the operation not completed in time and + // require user to schedule cancelation on completion port + this.CheckOperationCompletion(this); + + if (!this.IsCompleted && this.EndAsynchronousOperation != null && Interlocked.CompareExchange(ref this.isEndOperationCalled, True, False) == False) + { + // Call an end of asynchronous operation method if callback above returned false, this means + // it's us who is responsible for calling the end of asynchronous operation + this.EndAsynchronousOperation(this.CompletionResult); + } + } + + private Action CheckOperationCompletion { get; set; } + private AsyncCallback EndAsynchronousOperation { get; set; } + private IAsyncResult completionResult; + private int isEndOperationCalled; + + const int True = -1; + const int False = 0; + + #endregion + } + + // Defines a completion port which keeps track of asynchronous operations which can be run concurrently + // and results of their execution + public class CompletionPort + { + #region Constructors + + internal CompletionPort( + Action callback, + int totalOperationsCount, + int quorumOperationsCount, + TimeSpan timeout + ) + { + if (callback == null) + { + throw new ArgumentNullException("callback"); + } + + if (quorumOperationsCount > totalOperationsCount) + { + throw new ArgumentException("Invalid quorum operations count."); + } + + this.AsyncMachineResume = callback; + this.TotalOperationsCount = totalOperationsCount; + this.QuorumOperationsCount = quorumOperationsCount; + this.Timeout = timeout; + + this.OperationsLock = new object(); + this.Operations = new Dictionary(totalOperationsCount); + } + + internal CompletionPort(bool isSingleOperation) + { + Debug.Assert(isSingleOperation); + + this.IsSingleOperation = isSingleOperation; + } + + #endregion + + // Gets an asynchronous operation instance to be used for a given actor + public AsyncOperation this[object actor] + { + get + { + if (this.Operations == null) + { + throw new InvalidOperationException("CompletionPort.SingleOperation is not mutable."); + } + + AsyncOperation operation; + + lock (this.OperationsLock) + { + if (!this.Operations.TryGetValue(actor, out operation)) + { + // Check if async machine resume callback is already called, this means that + // this completion port is being reused which we don't allow + if (this.asyncMachineResumed == True) + { + throw new InvalidOperationException("Attempt to reuse a completion port."); + } + + operation = new AsyncOperation(this.CheckOperationCompletion); + this.Operations.Add(actor, operation); + } + } + + return operation; + } + } + + // Total number of operations in this completion token + public int TotalOperationsCount { get; private set; } + + // Number of operations to treat as a success to advance + public int QuorumOperationsCount { get; private set; } + + // Timeout indicating when wait on this completion port expires + public TimeSpan Timeout { get; private set; } + + // Provides a completion port to be used for single asynchronous operations + public static CompletionPort SingleOperation { get; private set; } + + #region Internal Members + + // Called by the async machine when this port is timed out and all operations have to marked as such + internal void TimeOutPort() + { + if (Interlocked.CompareExchange(ref this.asyncMachineResumed, True, False) == False) + { + // + // Async machine is not resumed at this point despite that some operations may be completed by this time. + // We'll mark all operations as incomplete and resume async machine execution so that user would schedule + // cancelation on all port operations. + // + + lock (this.OperationsLock) + { + foreach (var pair in this.Operations) + { + pair.Value.IsCompleted = false; + } + } + + Debug.Assert(this.AsyncMachineResume != null); + + this.AsyncMachineResume(); + } + } + + internal bool IsSingleOperation { get; private set; } + internal static readonly TimeSpan InfiniteWait = new TimeSpan(0, 0, 0, 0, -1); + + #endregion + + #region Private Members + + // Creates a singleton instance of completion port which can be used when async machine runs single + // asynchronous operations + static CompletionPort() + { + CompletionPort.SingleOperation = new CompletionPort(true); + } + + // Invoked by a completed asynchronous operation + private void CheckOperationCompletion(AsyncOperation operation) + { + // Mark async operation as completed if async machine is not resumed yet + operation.IsCompleted = this.asyncMachineResumed == False; + + if (operation.IsCompleted) + { + // + // Check if all or quorum (in case we're in quorum mode) number of operations have completed, + // in this case we need to notify the state machine that it can resume + // + + Debug.Assert(this.QuorumOperationsCount > 0); + + if (Interlocked.Increment(ref this.completedOperations) >= this.QuorumOperationsCount && + Interlocked.CompareExchange(ref this.asyncMachineResumed, True, False) == False) + { + Debug.Assert(this.AsyncMachineResume != null); + + this.AsyncMachineResume(); + } + } + } + + private Action AsyncMachineResume { get; set; } + private Dictionary Operations { get; set; } + private object OperationsLock { get; set; } + private int asyncMachineResumed; + private int completedOperations; + + const int True = -1; + const int False = 0; + + #endregion + } + + public delegate IEnumerable MachineEngine(AsyncMachine machine); + public delegate IEnumerable MachineEngine

(AsyncMachine machine, P param); + public delegate IEnumerable MachineEngine(AsyncMachine machine, P1 param1, P2 param2); + public delegate IEnumerable MachineEngine(AsyncMachine machine, P1 param1, P2 param2, P3 param3); + public delegate IEnumerable MachineEngine(AsyncMachine machine, P1 param1, P2 param2, P3 param3, P4 param4); + public delegate IEnumerable MachineEngine(AsyncMachine machine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5); + public delegate IEnumerable MachineEngine(AsyncMachine machine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6); + public delegate IEnumerable MachineEngine(AsyncMachine machine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, P7 param7); + public delegate IEnumerable MachineEngine(AsyncMachine machine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, P7 param7, P8 param8); + public delegate IEnumerable MachineEngine(AsyncMachine machine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, P7 param7, P8 param8, P9 param9); + public delegate IEnumerable MachineEngine(AsyncMachine machine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, P7 param7, P8 param8, P9 param9, P10 param10); + public delegate IEnumerable MachineEngine(AsyncMachine machine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, P7 param7, P8 param8, P9 param9, P10 param10, P11 param11); + + // General purpose async machine which supports single (optimized) and multiple + // asynchronous operations + public class AsyncMachine : IAsyncResult, IDisposable + { + #region Helper static methods + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngine engine, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine

(MachineEngine

engine, P param, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngine engine, P1 param1, P2 param2, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param1, param2)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngine engine, P1 param1, P2 param2, P3 param3, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param1, param2, param3)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngine engine, P1 param1, P2 param2, P3 param3, P4 param4, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param1, param2, param3, param4)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngine engine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param1, param2, param3, param4, param5)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngine engine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param1, param2, param3, param4, param5, param6)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngine engine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, P7 param7, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param1, param2, param3, param4, param5, param6, param7)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngine engine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, P7 param7, P8 param8, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param1, param2, param3, param4, param5, param6, param7, param8)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngine engine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, P7 param7, P8 param8, P9 param9, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param1, param2, param3, param4, param5, param6, param7, param8, param9)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngine engine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, P7 param7, P8 param8, P9 param9, P10 param10, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngine engine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, P7 param7, P8 param8, P9 param9, P10 param10, P11 param11, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11)); + return machine; + } + + + public static void EndAsyncMachine(IAsyncResult result) + { + using (AsyncMachine machine = (AsyncMachine)result) + { + machine.End(); + } + } + + #endregion + + #region Constructors + + public AsyncMachine(AsyncCallback callback, object state) + { + this.asyncMachineCompletionCallback = callback; + this.AsyncState = state; + + this.CompletionCallback = this.SingleOperationCompletionCallback; + this.moveNextLock = new object(); + } + + #endregion + + + // Create completion port with infinite timeout + public CompletionPort CreateCompletionPort(int totalOperationsCount) + { + return CreateCompletionPort(totalOperationsCount, CompletionPort.InfiniteWait); + } + + // Create completion port with specified timeout + public CompletionPort CreateCompletionPort(int totalOperationsCount, TimeSpan timeout) + { + return CreateCompletionPort(totalOperationsCount, totalOperationsCount, timeout); + } + + // Creates completion port in the quorum mode with a specified timeout (you can use CompletionPort.InfiniteWait + // for infinite timeout) + public CompletionPort CreateCompletionPort(int totalOperationsCount, int quorumOperationsCount) + { + return CreateCompletionPort( + totalOperationsCount, + quorumOperationsCount, + CompletionPort.InfiniteWait); + } + + // Creates completion port in the quorum mode with a specified timeout (you can use CompletionPort.InfiniteWait + // for infinite timeout) + public CompletionPort CreateCompletionPort(int totalOperationsCount, int quorumOperationsCount, TimeSpan timeout) + { + if (this.Enumerator == null) + { + throw new InvalidOperationException("Async machine either hasn't started yet or has already completed."); + } + + return new CompletionPort( + this.AsyncMachineResumeCallback, + totalOperationsCount, + quorumOperationsCount, + timeout); + } + + public delegate void ExceptionCleanupDelegate(object sender, EventArgs e); + + // Event which is fired when exception cleanup is required + public event ExceptionCleanupDelegate ExceptionCleanup; + + // Starts executing of the async machine + public void Start(IEnumerable machine) + { + Debug.Assert(null == this.Enumerator || this.IsCompleted); + + if (null == machine) + { + throw new ArgumentNullException("machine"); + } + + this.Enumerator = machine.GetEnumerator(); + this.IsCompleted = false; + this.Error = null; + + if (this.waitHandle != null) + { + this.waitHandle.Reset(); + } + + lock (this.moveNextLock) + { + this.CompletedSynchronously = true; + + this.MoveNext(); + + if (!this.IsCompleted) + { + this.CompletedSynchronously = false; + } + } + } + + // Ends the async machine and waits for its completion if necessary + public void End() + { + if (!this.IsCompleted) + { + this.AsyncWaitHandle.WaitOne(); + } + + Debug.Assert(this.IsCompleted); + + if (this.Error != null) + { + throw this.Error.PrepareServerStackForRethrow(); + } + } + + // A callback supplied to a single asynchronous operation to get notified when it's completed + public AsyncCallback CompletionCallback { get; private set; } + + // A result of completed single asynchronous operation when AsyncMachine.CompletionCallback is used + public IAsyncResult CompletionResult + { + get + { + if (this.completionResult == null) + { + throw new InvalidOperationException("Completion result is not available yet."); + } + + return this.completionResult; + } + } + + #region IAsyncResult Members + + public object AsyncState { get; private set; } + + public WaitHandle AsyncWaitHandle + { + get + { + if (null == this.waitHandle) + { + lock (this.moveNextLock) + { + if (null == this.waitHandle) + { + this.waitHandle = new EventWaitHandle(this.IsCompleted, EventResetMode.ManualReset); + } + } + } + + return this.waitHandle; + } + } + + public bool CompletedSynchronously { get; private set; } + + public bool IsCompleted { get; private set; } + + #endregion + + #region IDisposable Members + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool isDisposing) + { + if(isDisposing) + { + if (this.waitHandle != null) + { + // TODO: CLU + this.waitHandle.Dispose(); + //this.waitHandle.Close(); + this.waitHandle = null; + } + + if (this.WakeUpTimer != null) + { + this.WakeUpTimer.Dispose(); + this.WakeUpTimer = null; + } + } + } + + #endregion + + #region Private Members + + private void MoveNext() + { + lock (this.moveNextLock) + { + using (this.EnterThreadContext()) + { + this.MoveNextInternal(); + } + } + } + + private void MoveNextInternal() + { + Debug.Assert(this.Enumerator != null); + Debug.Assert(!this.IsCompleted); + Debug.Assert(null == this.Error); + + if (!this.InsideMoveNext) + { + try + { + try + { + this.InsideMoveNext = true; + this.IsCompleted = !this.Enumerator.MoveNext(); + + if (!this.IsCompleted) + { + if (this.Enumerator.Current == null) + { + throw new InvalidOperationException("Completion port for current iteration is null."); + } + + if (!this.Enumerator.Current.IsSingleOperation) + { + if (this.Enumerator.Current.TotalOperationsCount == 0) + { + // No operations are specified on the port, do another machine iteration + this.InsideMoveNext = false; + this.MoveNext(); + return; + } + else + { + // Check if we should schedule a wake up for the current completion port. That needs to happen + // when we use non-optimized completion port with a timeout + if (!this.Enumerator.Current.Timeout.Equals(CompletionPort.InfiniteWait)) + { + if (this.WakeUpTimer != null) + { + this.WakeUpTimer.Dispose(); + } + + // + // BUGBUG: Too expensive. Use single active task to implement timers + // + this.WakeUpTimer = new Timer( + this.CompletionPortTimeoutCallback, + this.Enumerator.Current, + this.Enumerator.Current.Timeout, + CompletionPort.InfiniteWait); + } + } + } + } + } + finally + { + this.InsideMoveNext = false; + } + } + catch (Exception e) + { + if (e.IsCritical()) + { + Trace.TraceError("Failed to forward message: {0}", e); + throw; + } + + Trace.TraceWarning("Failed to forward message: {0}", e); + + this.Error = e; + this.IsCompleted = true; + + if (this.ExceptionCleanup != null) + { + this.ExceptionCleanup(this, EventArgs.Empty); + } + } + + this.CompletedMoveNext(); + + if (this.AsyncMethodCompletedSynchronously && !this.IsCompleted) + { + this.AsyncMethodCompletedSynchronously = false; + + this.MoveNext(); + } + } + else + { + // + // Call to asynchronous I/O was completed synchronously (i.e. this thread is currently executing MoveNext down the stack) + // + + Debug.Assert(!this.AsyncMethodCompletedSynchronously); + + this.AsyncMethodCompletedSynchronously = true; + } + } + + private void CompletedMoveNext() + { + if (this.IsCompleted) + { + this.Enumerator = null; + + if (null != this.waitHandle) + { + this.waitHandle.Set(); + } + + if (this.asyncMachineCompletionCallback != null) + { + this.asyncMachineCompletionCallback((IAsyncResult)this); + } + } + } + + // Called by the WakeUpTimer when a scheduled timeout wait for the given completion + // port is expired + private void CompletionPortTimeoutCallback(object state) + { + Debug.Assert(state != null); + + CompletionPort port = (CompletionPort)state; + + port.TimeOutPort(); + } + + // Called by the completion port when quorum of asynchronous operations are completed + private void AsyncMachineResumeCallback() + { + // Check if quorum was reached with all operations finishing synchronously + if (!this.InsideMoveNext) + { + IEnumerator enumerator = this.Enumerator; + + // It should never happen that this callback is called for CompletionPort with single operation + if (enumerator != null && enumerator.Current != null && enumerator.Current.IsSingleOperation) + { + throw new InvalidOperationException("CompletionPort.SingleOperation was used."); + } + } + + this.MoveNext(); + } + + // Called by the completed single asynchronous operation + private void SingleOperationCompletionCallback(IAsyncResult asyncResult) + { + if (asyncResult == null) + { + throw new ArgumentNullException("asyncResult"); + } + + Debug.Assert(this.InsideMoveNext || this.Enumerator != null); + Debug.Assert(this.InsideMoveNext || this.Enumerator.Current != null); + + this.completionResult = asyncResult; + + this.MoveNext(); + } + + #region Thread Context helpers + + private IDisposable EnterThreadContext() + { + return null; + } + #endregion + + private IEnumerator Enumerator { get; set; } + private Exception Error { get; set; } + private bool InsideMoveNext { get; set; } + private bool AsyncMethodCompletedSynchronously { get; set; } + private Timer WakeUpTimer { get; set; } + private EventWaitHandle waitHandle; + private IAsyncResult completionResult; + private readonly AsyncCallback asyncMachineCompletionCallback; + private readonly object moveNextLock; + #endregion + } + + #region Parameterized async machines + + public delegate IEnumerable MachineEngineT(AsyncMachine machine); + public delegate IEnumerable MachineEngineT(AsyncMachine machine, P param); + public delegate IEnumerable MachineEngineT(AsyncMachine machine, P1 param1, P2 param2); + public delegate IEnumerable MachineEngineT(AsyncMachine machine, P1 param1, P2 param2, P3 param3); + public delegate IEnumerable MachineEngineT(AsyncMachine machine, P1 param1, P2 param2, P3 param3, P4 param4); + public delegate IEnumerable MachineEngineT(AsyncMachine machine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5); + public delegate IEnumerable MachineEngineT(AsyncMachine machine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6); + public delegate IEnumerable MachineEngineT(AsyncMachine machine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, P7 param7); + public delegate IEnumerable MachineEngineT(AsyncMachine machine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, P7 param7, P8 param8); + public delegate IEnumerable MachineEngineT(AsyncMachine machine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, P7 param7, P8 param8, P9 param9); + public delegate IEnumerable MachineEngineT(AsyncMachine machine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, P7 param7, P8 param8, P9 param9, P10 param10); + public delegate IEnumerable MachineEngineT(AsyncMachine machine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, P7 param7, P8 param8, P9 param9, P10 param10, P11 param11); + + public sealed class AsyncMachine : AsyncMachine + { + #region Helper static methods + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngineT engine, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine

(MachineEngineT engine, P param, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngineT engine, P1 param1, P2 param2, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param1, param2)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngineT engine, P1 param1, P2 param2, P3 param3, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param1, param2, param3)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngineT engine, P1 param1, P2 param2, P3 param3, P4 param4, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param1, param2, param3, param4)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngineT engine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param1, param2, param3, param4, param5)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngineT engine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param1, param2, param3, param4, param5, param6)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngineT engine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, P7 param7, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param1, param2, param3, param4, param5, param6, param7)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngineT engine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, P7 param7, P8 param8, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param1, param2, param3, param4, param5, param6, param7, param8)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngineT engine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, P7 param7, P8 param8, P9 param9, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param1, param2, param3, param4, param5, param6, param7, param8, param9)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngineT engine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, P7 param7, P8 param8, P9 param9, P10 param10, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngineT engine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, P7 param7, P8 param8, P9 param9, P10 param10, P11 param11, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11)); + return machine; + } + + public new static T EndAsyncMachine(IAsyncResult result) + { + using (AsyncMachine machine = (AsyncMachine)result) + { + machine.End(); + return machine.ParameterValue; + } + } + + #endregion + + public AsyncMachine() + : this(null, null) + { + } + + public AsyncMachine(AsyncCallback callback, object state) + : base(callback, state) + { + } + + public T ParameterValue { get; set; } + } + + public delegate IEnumerable MachineEngineRT(AsyncMachine machine); + public delegate IEnumerable MachineEngineRT(AsyncMachine machine, P param); + public delegate IEnumerable MachineEngineRT(AsyncMachine machine, P1 param1, P2 param2); + public delegate IEnumerable MachineEngineRT(AsyncMachine machine, P1 param1, P2 param2, P3 param3); + public delegate IEnumerable MachineEngineRT(AsyncMachine machine, P1 param1, P2 param2, P3 param3, P4 param4); + public delegate IEnumerable MachineEngineRT(AsyncMachine machine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5); + public delegate IEnumerable MachineEngineRT(AsyncMachine machine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6); + public delegate IEnumerable MachineEngineRT(AsyncMachine machine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, P7 param7); + public delegate IEnumerable MachineEngineRT(AsyncMachine machine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, P7 param7, P8 param8); + public delegate IEnumerable MachineEngineRT(AsyncMachine machine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, P7 param7, P8 param8, P9 param9); + public delegate IEnumerable MachineEngineRT(AsyncMachine machine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, P7 param7, P8 param8, P9 param9, P10 param10); + public delegate IEnumerable MachineEngineRT(AsyncMachine machine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, P7 param7, P8 param8, P9 param9, P10 param10, P11 param11); + + public sealed class AsyncMachine : AsyncMachine + { + #region Helper static methods + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngineRT engine, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine

(MachineEngineRT engine, P param, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngineRT engine, P1 param1, P2 param2, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param1, param2)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngineRT engine, P1 param1, P2 param2, P3 param3, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param1, param2, param3)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngineRT engine, P1 param1, P2 param2, P3 param3, P4 param4, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param1, param2, param3, param4)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngineRT engine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param1, param2, param3, param4, param5)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngineRT engine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param1, param2, param3, param4, param5, param6)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngineRT engine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, P7 param7, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param1, param2, param3, param4, param5, param6, param7)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngineRT engine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, P7 param7, P8 param8, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param1, param2, param3, param4, param5, param6, param7, param8)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngineRT engine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, P7 param7, P8 param8, P9 param9, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param1, param2, param3, param4, param5, param6, param7, param8, param9)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngineRT engine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, P7 param7, P8 param8, P9 param9, P10 param10, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10)); + return machine; + } + + [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "EndAsyncMachine does the disposing")] + public static IAsyncResult BeginAsyncMachine(MachineEngineRT engine, P1 param1, P2 param2, P3 param3, P4 param4, P5 param5, P6 param6, P7 param7, P8 param8, P9 param9, P10 param10, P11 param11, AsyncCallback callback, object asyncState) + { + AsyncMachine machine = new AsyncMachine(callback, asyncState); + machine.Start(engine(machine, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11)); + return machine; + } + + public static TReturn EndAsyncMachine(IAsyncResult result, out T parameterValue) + { + using (AsyncMachine machine = (AsyncMachine)result) + { + machine.End(); + parameterValue = machine.ParameterValue; + return machine.ReturnValue; + } + } + + #endregion + + public AsyncMachine() + : this(null, null) + { + } + + public AsyncMachine(AsyncCallback callback, object state) + : base(callback, state) + { + } + + public TReturn ReturnValue { get; set; } + public T ParameterValue { get; set; } + } + + #endregion +} diff --git a/src/CLU/VhdManagement/Async/ExceptionExtension.cs b/src/CLU/VhdManagement/Async/ExceptionExtension.cs new file mode 100644 index 000000000000..32462a82c14d --- /dev/null +++ b/src/CLU/VhdManagement/Async/ExceptionExtension.cs @@ -0,0 +1,66 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Diagnostics; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Threading; + +namespace Microsoft.WindowsAzure.Commands.Tools.Common.General +{ + public static class ExceptionExtensions + { + private static readonly MethodInfo methodPrepForRemoting = typeof(Exception).GetMethod("PrepForRemoting", BindingFlags.NonPublic | BindingFlags.Instance); + + public static bool IsCritical( + this Exception exception + ) + { + Debug.Assert(null != exception); + + if (null == exception) + { + throw new ArgumentNullException("exception"); + } + + // TODO: CLU + return ( + (exception is OutOfMemoryException) || + (exception is SEHException) + ); + /* + return ( + (exception is AccessViolationException) || + (exception is InsufficientMemoryException) || + (exception is OutOfMemoryException) || + (exception is SEHException) || + (exception is StackOverflowException) || + (exception is ThreadAbortException) + ); + */ + } + + public static Exception PrepareServerStackForRethrow( + this Exception exception + ) + { + Debug.Assert(null != exception); + + methodPrepForRemoting.Invoke(exception, new object[0]); + + return exception; + } + } +} diff --git a/src/CLU/VhdManagement/EntryStub.cs b/src/CLU/VhdManagement/EntryStub.cs new file mode 100644 index 000000000000..7dd4fdc0015c --- /dev/null +++ b/src/CLU/VhdManagement/EntryStub.cs @@ -0,0 +1,24 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd +{ + public class EntryStub + { + public static void Main(string[] args) + { + // empty entry point + } + } +} diff --git a/src/CLU/VhdManagement/MSSharedLibKey.snk b/src/CLU/VhdManagement/MSSharedLibKey.snk new file mode 100644 index 000000000000..695f1b38774e Binary files /dev/null and b/src/CLU/VhdManagement/MSSharedLibKey.snk differ diff --git a/src/CLU/VhdManagement/Model/AttributeHelper.cs b/src/CLU/VhdManagement/Model/AttributeHelper.cs new file mode 100644 index 000000000000..058e4c1b9a32 --- /dev/null +++ b/src/CLU/VhdManagement/Model/AttributeHelper.cs @@ -0,0 +1,111 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Linq; +using System.Linq.Expressions; +using System.Reflection; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model +{ + public class AttributeHelper + { + private readonly Type type; + public AttributeHelper() + { + type = typeof (T); + } + + public VhdEntityAttribute GetEntityAttribute() + { + // TODO: CLU + var attributes = type.GetTypeInfo().GetCustomAttributes(typeof(VhdEntityAttribute), false); + if (attributes.Count() == 0) + { + throw new InvalidOperationException(String.Format("Entity must have the attribute:{0}", typeof(VhdEntityAttribute).Name)); + } + return (VhdEntityAttribute)attributes.ElementAt(0); + /* + var attributes = type.GetCustomAttributes(typeof(VhdEntityAttribute), false); + if (attributes.Length == 0) + { + throw new InvalidOperationException(String.Format("Entity must have the attribute:{0}", typeof(VhdEntityAttribute).Name)); + } + return (VhdEntityAttribute)attributes[0]; + */ + } + + public VhdPropertyAttribute GetAttribute(Expression> propertyNameProvider) + { + MemberExpression me; + if (propertyNameProvider.Body is UnaryExpression) + { + me = ((UnaryExpression)propertyNameProvider.Body).Operand as MemberExpression; + } + else + { + me = propertyNameProvider.Body as MemberExpression; + } + + if (me == null || me.Expression.Type != type) + { + throw new InvalidOperationException("Not a valid expression, must be a VhdFooter property access"); + } + + var propertyName = me.Member.Name; + + var attributes = from p in type.GetProperties(BindingFlags.Instance | BindingFlags.Public) + let vhdPropertyAttributes = p.GetCustomAttributes(typeof(VhdPropertyAttribute), false) + // TODO: CLU + let exists = vhdPropertyAttributes.Count() > 0 + // let exists = vhdPropertyAttributes.Length > 0 + where p.Name == propertyName + // TODO: CLU + select (VhdPropertyAttribute)(vhdPropertyAttributes.ElementAt(0)); + // select (VhdPropertyAttribute)(vhdPropertyAttributes[0]); + return attributes.FirstOrDefault(); + } + + public VhdPropertyAttribute GetAttribute(Expression> propertyNameProvider) + { + MemberExpression me; + if (propertyNameProvider.Body is UnaryExpression) + { + me = ((UnaryExpression)propertyNameProvider.Body).Operand as MemberExpression; + } + else + { + me = propertyNameProvider.Body as MemberExpression; + } + + if (me == null || me.Expression.Type != type) + { + throw new InvalidOperationException("Not a valid expression, must be a VhdFooter property access"); + } + + var propertyName = me.Member.Name; + + var attributes = from p in type.GetProperties(BindingFlags.Instance | BindingFlags.Public) + let vhdPropertyAttributes = p.GetCustomAttributes(typeof(VhdPropertyAttribute), false) + // TODO: CLU + let exists = vhdPropertyAttributes.Count() > 0 + // let exists = vhdPropertyAttributes.Length > 0 + where p.Name == propertyName + // TODO: CLU + select (VhdPropertyAttribute)(vhdPropertyAttributes.ElementAt(0)); + // select (VhdPropertyAttribute)(vhdPropertyAttributes[0]); + return attributes.FirstOrDefault(); + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/BitMap.cs b/src/CLU/VhdManagement/Model/BitMap.cs new file mode 100644 index 000000000000..109f5cf0077e --- /dev/null +++ b/src/CLU/VhdManagement/Model/BitMap.cs @@ -0,0 +1,38 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System.Collections; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model +{ + public class BitMap + { + public BitMap(BitArray data) + { + this.Data = data; + Covered = CheckIfCovered(); + } + + public BitArray Data { get; private set; } + + public bool Covered { get; private set; } + + bool CheckIfCovered() + { + for (int i = 0; i < Data.Length; i++) + if (!Data[i]) return false; + return true; + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/Block.cs b/src/CLU/VhdManagement/Model/Block.cs new file mode 100644 index 000000000000..f46c24cd9f06 --- /dev/null +++ b/src/CLU/VhdManagement/Model/Block.cs @@ -0,0 +1,65 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using Microsoft.WindowsAzure.Commands.Tools.Vhd.Model.Persistence; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model +{ + public class Block + { + private readonly IBlockFactory blockFactory; + private byte[] data; + + public Block(IBlockFactory blockFactory) + { + this.blockFactory = blockFactory; + } + + public Guid VhdUniqueId { get; set; } + public uint BlockIndex { get; set; } + public BitMap BitMap { get; set; } + + public byte[] Data + { + get + { + if (data == null) + { + data = this.blockFactory.ReadBlockData(this); + } + return data; + } + } + + public long SectorCount + { + get { return this.LogicalRange.Length / VhdConstants.VHD_SECTOR_LENGTH; } + } + + public Sector GetSector(uint sector) + { + return this.blockFactory.GetSector(this, sector); + } + + public bool Empty { get; set; } + + public IndexRange LogicalRange { get; set; } + + public override string ToString() + { + return BlockIndex.ToString(); + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/BlockAllocationTable.cs b/src/CLU/VhdManagement/Model/BlockAllocationTable.cs new file mode 100644 index 000000000000..6a004f821c2e --- /dev/null +++ b/src/CLU/VhdManagement/Model/BlockAllocationTable.cs @@ -0,0 +1,61 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using Microsoft.WindowsAzure.Commands.Tools.Vhd.Model.Persistence; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model +{ + public class BlockAllocationTable + { + private readonly uint blockSize; + + public BlockAllocationTable(uint size, uint blockSize, uint[] bat) + { + this.blockSize = blockSize; + this.Size = size; + this.Data = bat; + } + + public uint Size { get; internal set; } + + private uint[] Data { get; set; } + + public long GetBlockAddress(uint block) + { + return GetBitMapAddress(block) + GetSectorPaddedBitmapSizeInBytes(); + } + + public long GetBitMapAddress(uint block) + { + return ((long)this.Data[block]) * VhdConstants.VHD_SECTOR_LENGTH; + } + + public int GetSectorPaddedBitmapSizeInBytes() + { + var sectorSpanOfBitMap = (double)GetBitmapSizeInBytes() / VhdConstants.VHD_SECTOR_LENGTH; + return (int)(Math.Ceiling(sectorSpanOfBitMap) * VhdConstants.VHD_SECTOR_LENGTH); + } + + public int GetBitmapSizeInBytes() + { + return (int)(blockSize / VhdConstants.VHD_SECTOR_LENGTH / 8); + } + + public bool HasData(uint block) + { + return block != VhdConstants.VHD_NO_DATA_INT && Data[block] != VhdConstants.VHD_NO_DATA_INT; + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/DiskGeometry.cs b/src/CLU/VhdManagement/Model/DiskGeometry.cs new file mode 100644 index 000000000000..1736edbdbd6d --- /dev/null +++ b/src/CLU/VhdManagement/Model/DiskGeometry.cs @@ -0,0 +1,122 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using Microsoft.WindowsAzure.Commands.Tools.Vhd.Model.Persistence; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model +{ + [VhdEntity(Size = 4)] + public class DiskGeometry + { + public static DiskGeometry CreateFromVirtualSize(long size) + { + long totalSectors = size / VhdConstants.VHD_SECTOR_LENGTH; + if (totalSectors > 65535 * 16 * 255) + { + totalSectors = 65535 * 16 * 255; + } + + int sectorsPerTrack; + int heads; + long cylinderTimesHeads; + if (totalSectors >= 65535 * 16 * 63) + { + sectorsPerTrack = 255; + heads = 16; + cylinderTimesHeads = totalSectors / sectorsPerTrack; + } + else + { + sectorsPerTrack = 17; + cylinderTimesHeads = totalSectors / sectorsPerTrack; + + heads = (int) ((cylinderTimesHeads + 1023) / 1024); + + if (heads < 4) + { + heads = 4; + } + if (cylinderTimesHeads >= (heads * 1024) || heads > 16) + { + sectorsPerTrack = 31; + heads = 16; + cylinderTimesHeads = totalSectors / sectorsPerTrack; + } + if (cylinderTimesHeads >= (heads * 1024)) + { + sectorsPerTrack = 63; + heads = 16; + cylinderTimesHeads = totalSectors / sectorsPerTrack; + } + } + long cylinders = cylinderTimesHeads/heads; + + return new DiskGeometry + { + Cylinder = (short) cylinders, + Heads = (byte) heads, + Sectors = (byte) sectorsPerTrack + }; + } + + [VhdProperty(Offset = 0, Size = 2)] + public short Cylinder { get; set; } + + [VhdProperty(Offset = 2, Size = 1)] + public byte Heads { get; set; } + + [VhdProperty(Offset = 3, Size = 1)] + public byte Sectors { get; set; } + + public DiskGeometry CreateCopy() + { + return new DiskGeometry + { + Cylinder = this.Cylinder, + Heads = this.Heads, + Sectors = this.Sectors + }; + } + + public override string ToString() + { + return String.Format("Cylinder:{0}, Heads:{1}, Sector:{2}", Cylinder, Heads, Sectors); + } + + public override bool Equals(object obj) + { + if (ReferenceEquals(null, obj)) return false; + if (ReferenceEquals(this, obj)) return true; + if (obj.GetType() != typeof (DiskGeometry)) return false; + return Equals((DiskGeometry) obj); + } + + private bool Equals(DiskGeometry other) + { + return other.Cylinder == Cylinder && other.Heads == Heads && other.Sectors == Sectors; + } + + public override int GetHashCode() + { + unchecked + { + int result = Cylinder.GetHashCode(); + result = (result*397) ^ Heads.GetHashCode(); + result = (result*397) ^ Sectors.GetHashCode(); + return result; + } + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/DiskType.cs b/src/CLU/VhdManagement/Model/DiskType.cs new file mode 100644 index 000000000000..26ba48bc5835 --- /dev/null +++ b/src/CLU/VhdManagement/Model/DiskType.cs @@ -0,0 +1,24 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model +{ + public enum DiskType + { + None = 0, + Fixed = 2, + Dynamic = 3, + Differencing = 4 + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/HostOsType.cs b/src/CLU/VhdManagement/Model/HostOsType.cs new file mode 100644 index 000000000000..b65b44b61104 --- /dev/null +++ b/src/CLU/VhdManagement/Model/HostOsType.cs @@ -0,0 +1,22 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model +{ + public enum HostOsType + { + Windows = 0x5769326B, + Macintosh = 0x4D616320 + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/IndexRange.cs b/src/CLU/VhdManagement/Model/IndexRange.cs new file mode 100644 index 000000000000..fedd6d199337 --- /dev/null +++ b/src/CLU/VhdManagement/Model/IndexRange.cs @@ -0,0 +1,214 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model +{ + public class IndexRange : IEquatable + { + static IndexRangeComparer comparer = new IndexRangeComparer(); + + public static IEnumerable SubstractRanges(IEnumerable source, IEnumerable ranges) + { + var onlyInSource = source.Where(e => !ranges.Any(r => r.Intersects(e))); + + var irs = + from ur in ranges + from r in source + where r.Intersects(ur) + from ir in r.Subtract(ur) + select ir; + + var result = irs.Distinct(new IndexRangeComparer()).ToList(); + result.AddRange(onlyInSource); + result.Sort((r1, r2) => r1.CompareTo(r2)); + return result; + } + + public static IndexRange FromLength(long startIndex, long length) + { + return new IndexRange(startIndex, startIndex + length - 1); + } + + public IndexRange(long startIndex, long endIndex) + { + this.StartIndex = startIndex; + this.EndIndex = endIndex; + } + + public bool After(IndexRange range) + { + if(Intersects(range)) + { + return false; + } + + return (this.StartIndex > range.EndIndex); + } + + public IEnumerable Subtract(IndexRange range) + { + if(this.Equals(range)) + { + return new List(); + } + if(!this.Intersects(range)) + { + return new List {this}; + } + var intersection = this.Intersection(range); + if(this.Equals(intersection)) + { + return new List(); + } + if(intersection.StartIndex == this.StartIndex) + { + return new List {new IndexRange(intersection.EndIndex + 1, this.EndIndex)}; + } + if (intersection.EndIndex == this.EndIndex) + { + return new List { new IndexRange(this.StartIndex, intersection.StartIndex - 1)}; + } + return new List + { + new IndexRange(this.StartIndex, intersection.StartIndex - 1), + new IndexRange(intersection.EndIndex + 1, this.EndIndex) + }; + } + + public bool Abuts(IndexRange range) + { + return !this.Intersects(range) && this.Gap(range) == null; + } + + public IndexRange Gap(IndexRange range) + { + if(this.Intersects(range)) + return null; + if(this.CompareTo(range) > 0) + { + var r = new IndexRange(range.EndIndex + 1, this.StartIndex - 1); + if(r.Length <= 0) + return null; + return r; + } + var result = new IndexRange(this.EndIndex + 1, range.StartIndex - 1); + if(result.Length <= 0) + return null; + return result; + } + + public int CompareTo(IndexRange range) + { + return this.StartIndex != range.StartIndex ? + this.StartIndex.CompareTo(range.StartIndex) : + this.EndIndex.CompareTo(range.EndIndex); + } + + public IndexRange Merge(IndexRange range) + { + if(!this.Abuts(range)) + { + throw new ArgumentOutOfRangeException("range", "Ranges must be adjacent."); + } + if(this.CompareTo(range) > 0) + { + return new IndexRange(range.StartIndex, this.EndIndex); + } + return new IndexRange(this.StartIndex, range.EndIndex); + } + + public IEnumerable PartitionBy(int size) + { + if(this.Length <= size) + { + return new List {this}; + } + var result = new List(); + long count = this.Length/size; + long remainder = this.Length%size; + for (long i = 0; i < count; i++) + { + result.Add(IndexRange.FromLength(this.StartIndex + i*size, size)); + } + if(remainder != 0) + { + result.Add(IndexRange.FromLength(this.StartIndex + count*size, remainder)); + } + return result; + } + + public IndexRange Intersection(IndexRange range) + { + if(!this.Intersects(range)) + { + return null; + } + var start = Math.Max(range.StartIndex, this.StartIndex); + var end = Math.Min(range.EndIndex, this.EndIndex); + + return new IndexRange(start, end); + } + + public bool Intersects(IndexRange range) + { + var start = Math.Max(range.StartIndex, this.StartIndex); + var end = Math.Min(range.EndIndex, this.EndIndex); + return start <= end; + } + + public bool Includes(IndexRange range) + { + return this.Includes(range.StartIndex) && this.Includes(range.EndIndex); + } + + public long EndIndex { get; private set; } + + public long StartIndex { get; private set; } + + public bool Includes(long value) + { + return value >= StartIndex && value <= EndIndex; + } + + public long Length + { + get { return EndIndex - StartIndex + 1; } + } + + public bool Equals(IndexRange other) + { + return other != null && this.StartIndex == other.StartIndex && this.EndIndex == other.EndIndex; + } + + public override string ToString() + { + return String.Format("[{0},{1}]", StartIndex, EndIndex); + } + + public override bool Equals(object obj) + { + var range = obj as IndexRange; + return range != null && comparer.Equals(this, range); + } + + public override int GetHashCode() + { + return comparer.GetHashCode(this); + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/IndexRangeComparer.cs b/src/CLU/VhdManagement/Model/IndexRangeComparer.cs new file mode 100644 index 000000000000..342014e82d87 --- /dev/null +++ b/src/CLU/VhdManagement/Model/IndexRangeComparer.cs @@ -0,0 +1,33 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System.Collections.Generic; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model +{ + public class IndexRangeComparer : IEqualityComparer + { + public bool Equals(IndexRange first, IndexRange second) + { + return first.Equals(second); + } + + public int GetHashCode(IndexRange range) + { + var hash = 17 + range.StartIndex.GetHashCode(); + hash = hash * 17 + range.EndIndex.GetHashCode(); + return hash; + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/Model.cs b/src/CLU/VhdManagement/Model/Model.cs new file mode 100644 index 000000000000..d357ee0bff03 --- /dev/null +++ b/src/CLU/VhdManagement/Model/Model.cs @@ -0,0 +1,165 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.IO; +using System.Runtime.Serialization; +using Microsoft.WindowsAzure.Commands.Tools.Common.General; +using Microsoft.WindowsAzure.Commands.Tools.Vhd.Model.Persistence; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model +{ + [Flags] + public enum VhdValidationType + { + None, + IsVhd, + FixedDisk, + } + + public class VhdValidator + { + public static IList Validate(VhdValidationType validation, string path) + { + var fileFactory = new VhdFileFactory(); + VhdFile vhdFile = null; + Exception exception = null; + try + { + vhdFile = fileFactory.Create(path); + } + catch (VhdParsingException e) + { + exception = e; + } + + return DoValidate(validation, vhdFile, exception); + } + + public static IList Validate(VhdValidationType validation, Stream vhdStream) + { + + var fileFactory = new VhdFileFactory(); + VhdFile vhdFile = null; + Exception exception = null; + try + { + vhdFile = fileFactory.Create(vhdStream); + } + catch (VhdParsingException e) + { + exception = e; + } + + return DoValidate(validation, vhdFile, exception); + } + + private static IList DoValidate(VhdValidationType validation, VhdFile vhdFile, Exception exception) + { + var result = new List(); + + if ((validation & VhdValidationType.IsVhd) == VhdValidationType.IsVhd) + { + var validationResult = new VhdValidationResult + { + ValidationType = VhdValidationType.IsVhd + }; + if (vhdFile == null) + { + validationResult.ErrorCode = 1000; + validationResult.Error = exception; + } + result.Add(validationResult); + } + + if ((validation & VhdValidationType.FixedDisk) == VhdValidationType.FixedDisk) + { + var validationResult = new VhdValidationResult + { + ValidationType = VhdValidationType.FixedDisk + }; + if (vhdFile == null || vhdFile.Footer.DiskType != DiskType.Fixed) + { + validationResult.ErrorCode = 1001; + } + result.Add(validationResult); + } + return result; + } + + public static IAsyncResult BeginValidate(VhdValidationType validation, Stream vhdStream, AsyncCallback callback, object state) + { + return AsyncMachine>.BeginAsyncMachine(ValidateAsync, validation, vhdStream, callback, state); + } + + public static IList EndValidate(IAsyncResult result) + { + return AsyncMachine>.EndAsyncMachine(result); + } + + private static IEnumerable ValidateAsync(AsyncMachine> machine, VhdValidationType validation, Stream vhdStream) + { + var result = new List(); + + var fileFactory = new VhdFileFactory(); + + fileFactory.BeginCreate(vhdStream, machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + + VhdFile vhdFile = null; + Exception exception = null; + try + { + vhdFile = fileFactory.EndCreate(machine.CompletionResult); + } + catch (VhdParsingException e) + { + exception = e; + } + + machine.ParameterValue = DoValidate(validation, vhdFile, exception); + } + } + + public class VhdValidationResult + { + public int ErrorCode { get; set; } + public VhdValidationType ValidationType{ get; set; } + public Exception Error { get; set; } + } + // TODO: CLU + //[Serializable] + public class VhdParsingException : Exception + { + public VhdParsingException() + { + } + + public VhdParsingException(string message) : base(message) + { + } + + public VhdParsingException(string message, Exception innerException) : base(message, innerException) + { + } + + // TODO: CLU + /* + protected VhdParsingException(SerializationInfo info, StreamingContext context) : base(info, context) + { + } + */ + } +} diff --git a/src/CLU/VhdManagement/Model/ParentLocator.cs b/src/CLU/VhdManagement/Model/ParentLocator.cs new file mode 100644 index 000000000000..3a0b0983d8c1 --- /dev/null +++ b/src/CLU/VhdManagement/Model/ParentLocator.cs @@ -0,0 +1,39 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model +{ + + + [VhdEntity(Size = 24)] + public class ParentLocator + { + [VhdProperty(Offset = 0, Size = 4)] + public PlatformCode PlatformCode { get; set; } + + [VhdProperty(Offset = 4, Size = 4)] + public int PlatformDataSpace { get; set; } + + [VhdProperty(Offset = 8, Size = 4)] + public int PlatformDataLength { get; set; } + + [VhdProperty(Offset = 12, Size = 4)] + public int Reserved { get; set; } + + [VhdProperty(Offset = 16, Size = 8)] + public long PlatformDataOffset { get; set; } + + public string PlatformSpecificFileLocator { get; set; } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/Persistence/AbstractDiskBlockFactory.cs b/src/CLU/VhdManagement/Model/Persistence/AbstractDiskBlockFactory.cs new file mode 100644 index 000000000000..0d15d38f45e7 --- /dev/null +++ b/src/CLU/VhdManagement/Model/Persistence/AbstractDiskBlockFactory.cs @@ -0,0 +1,73 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model.Persistence +{ + public abstract class AbstractDiskBlockFactory : IBlockFactory + { + protected readonly VhdFile vhdFile; + private byte[] emptyBlockData; + + protected AbstractDiskBlockFactory(VhdFile vhdFile) + { + this.vhdFile = vhdFile; + } + + public abstract Block Create(uint block); + public abstract Sector GetSector(Block block, uint sector); + + public IndexRange GetFooterRange() + { + return IndexRange.FromLength(this.GetBlockSize() * this.BlockCount, VhdConstants.VHD_FOOTER_SIZE); + } + + public byte[] ReadBlockData(Block block) + { + if (!this.HasData(block.BlockIndex)) + { + if (emptyBlockData == null) + { + emptyBlockData = new byte[(int)GetBlockSize()]; + Array.Clear(emptyBlockData, 0, emptyBlockData.Length); + } + return emptyBlockData; + } + return DoReadBlockData(block); + } + + protected abstract byte[] DoReadBlockData(Block block); + + public long BlockCount + { + get { return vhdFile.BlockAllocationTable.Size; } + } + + public bool HasData(uint blockIndex) + { + return vhdFile.BlockAllocationTable.HasData(blockIndex); + } + + public long GetBlockAddress(uint blockIndex) + { + return vhdFile.BlockAllocationTable.GetBlockAddress(blockIndex); + } + + public long GetBlockSize() + { + return vhdFile.Header.BlockSize; + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/Persistence/BitMapFactory.cs b/src/CLU/VhdManagement/Model/Persistence/BitMapFactory.cs new file mode 100644 index 000000000000..d25ff13a0d44 --- /dev/null +++ b/src/CLU/VhdManagement/Model/Persistence/BitMapFactory.cs @@ -0,0 +1,50 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Collections; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model.Persistence +{ + public class BitMapFactory + { + private readonly VhdFile vhdFile; + + public BitMapFactory(VhdFile vhdFile) + { + this.vhdFile = vhdFile; + } + + public BitMap Create(uint block) + { + var bitMapAddress = vhdFile.BlockAllocationTable.GetBitMapAddress(block); + var bitmapSizeInBytes = vhdFile.BlockAllocationTable.GetBitmapSizeInBytes(); + var bytes = vhdFile.DataReader.ReadBytes(bitMapAddress, bitmapSizeInBytes); + ReverseBitsIfLittleEndian(bytes); + return new BitMap(new BitArray(bytes)); + } + + private static void ReverseBitsIfLittleEndian(byte[] bytes) + { + if (BitConverter.IsLittleEndian) + { + for (int bit = 0; bit < bytes.Length; bit++) + { + // reverse the bits due to quirky BitArray + bytes[bit] = (byte)(((bytes[bit] * 0x80200802UL) & 0x0884422110UL) * 0x0101010101UL >> 32); + } + } + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/Persistence/BlockAllocationTableFactory.cs b/src/CLU/VhdManagement/Model/Persistence/BlockAllocationTableFactory.cs new file mode 100644 index 000000000000..170c19c3a1ba --- /dev/null +++ b/src/CLU/VhdManagement/Model/Persistence/BlockAllocationTableFactory.cs @@ -0,0 +1,68 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Microsoft.WindowsAzure.Commands.Tools.Common.General; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model.Persistence +{ + public class BlockAllocationTableFactory + { + private readonly VhdDataReader dataReader; + private readonly VhdHeader header; + + public BlockAllocationTableFactory(VhdDataReader dataReader, VhdHeader header) + { + this.dataReader = dataReader; + this.header = header; + } + + public BlockAllocationTable Create() + { + dataReader.SetPosition(header.TableOffset); + + var bat = new uint[header.MaxTableEntries]; + for (int block = 0; block < header.MaxTableEntries; block++) + { + bat[block] = dataReader.ReadUInt32(); + } + return new BlockAllocationTable(header.MaxTableEntries, header.BlockSize, bat); + } + + public IAsyncResult BeginCreate( AsyncCallback callback, object state) + { + return AsyncMachine.BeginAsyncMachine(CreateAsync, callback, state); + } + + public BlockAllocationTable EndCreate(IAsyncResult result) + { + return AsyncMachine.EndAsyncMachine(result); + } + + private IEnumerable CreateAsync(AsyncMachine machine) + { + dataReader.SetPosition(header.TableOffset); + + var bat = new uint[header.MaxTableEntries]; + for (int block = 0; block < header.MaxTableEntries; block++) + { + dataReader.BeginReadUInt32(machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + bat[block] = dataReader.EndReadUInt32(machine.CompletionResult); + } + machine.ParameterValue = new BlockAllocationTable(header.MaxTableEntries, header.BlockSize, bat); + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/Persistence/CloudVhdFileCreator.cs b/src/CLU/VhdManagement/Model/Persistence/CloudVhdFileCreator.cs new file mode 100644 index 000000000000..6b93c4231975 --- /dev/null +++ b/src/CLU/VhdManagement/Model/Persistence/CloudVhdFileCreator.cs @@ -0,0 +1,66 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.IO; +using Microsoft.WindowsAzure.Commands.Tools.Common.General; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model.Persistence +{ + public class VhdFileCreator + { + public static void CreateFixedVhdFile(Stream destination, long virtualSize) + { + var footer = VhdFooterFactory.CreateFixedDiskFooter(virtualSize); + var serializer = new VhdFooterSerializer(footer); + var buffer = serializer.ToByteArray(); + destination.SetLength(virtualSize + VhdConstants.VHD_FOOTER_SIZE); + destination.Seek(-VhdConstants.VHD_FOOTER_SIZE, SeekOrigin.End); + + destination.Write(buffer, 0, buffer.Length); + destination.Flush(); + } + + public static IAsyncResult BeginCreateFixedVhdFile(Stream destination, long size, AsyncCallback callback, object state) + { + return AsyncMachine.BeginAsyncMachine(CreateFixedVhdFileAtAsync, destination, size, callback, state); + } + + public static void EndCreateFixedVhdFile(IAsyncResult result) + { + AsyncMachine.EndAsyncMachine(result); + } + + private static IEnumerable CreateFixedVhdFileAtAsync(AsyncMachine machine, Stream destination, long virtualSize) + { + var footer = VhdFooterFactory.CreateFixedDiskFooter(virtualSize); + var serializer = new VhdFooterSerializer(footer); + var buffer = serializer.ToByteArray(); + destination.SetLength(virtualSize + VhdConstants.VHD_FOOTER_SIZE); + destination.Seek(-VhdConstants.VHD_FOOTER_SIZE, SeekOrigin.End); + + // TODO: CLU + yield return CompletionPort.SingleOperation; + destination.Write(buffer, 0, buffer.Length); + destination.Flush(); + /* + destination.BeginWrite(buffer, 0, buffer.Length, machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + destination.EndWrite(machine.CompletionResult); + destination.Flush(); + */ + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/Persistence/DifferencingDiskBlockFactory.cs b/src/CLU/VhdManagement/Model/Persistence/DifferencingDiskBlockFactory.cs new file mode 100644 index 000000000000..54e92b537b6c --- /dev/null +++ b/src/CLU/VhdManagement/Model/Persistence/DifferencingDiskBlockFactory.cs @@ -0,0 +1,87 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model.Persistence +{ + public class DifferencingDiskBlockFactory : AbstractDiskBlockFactory + { + private BitMapFactory bitMapFactory; + private SectorFactory sectorFactory; + private IBlockFactory parentBlockFactory; + private Block cachedBlock; + + public DifferencingDiskBlockFactory(VhdFile vhdFile) : base(vhdFile) + { + this.bitMapFactory = new BitMapFactory(vhdFile); + this.sectorFactory = new SectorFactory(vhdFile, this); + this.parentBlockFactory = vhdFile.Parent.DiskType != DiskType.Fixed ? vhdFile.Parent.GetBlockFactory() : new FixedDiskBlockFactory(vhdFile.Parent, this.GetBlockSize()); + } + + public override Block Create(uint block) + { + if(!vhdFile.BlockAllocationTable.HasData(block)) + { + if(cachedBlock == null || cachedBlock.BlockIndex != block) + { + cachedBlock = parentBlockFactory.Create(block); + } + return cachedBlock; + } + + if (cachedBlock == null || cachedBlock.BlockIndex != block) + { + cachedBlock = new Block(this) + { + BlockIndex = block, + VhdUniqueId = this.vhdFile.Footer.UniqueId, + LogicalRange = IndexRange.FromLength(block * GetBlockSize(), vhdFile.Header.BlockSize), + BitMap = bitMapFactory.Create(block), + Empty = false + }; + } + return cachedBlock; + } + + public override Sector GetSector(Block block, uint sector) + { + if(block.Empty) + { + return this.sectorFactory.CreateEmptySector(block.BlockIndex, sector); + } + + if(block.BitMap != null && block.BitMap.Data[(int) sector]) + { + return this.sectorFactory.Create(block, sector); + } + + var parentBlock = parentBlockFactory.Create(block.BlockIndex); + return parentBlockFactory.GetSector(parentBlock, sector); + } + + protected override byte[] DoReadBlockData(Block block) + { + var result = new byte[GetBlockSize()]; + int index = 0; + for (int i = 0; i < block.SectorCount; i++) + { + var sector = block.GetSector((uint)i); + Buffer.BlockCopy(sector.Data, 0, result, index, sector.Data.Length); + index += sector.Data.Length; + } + return result; + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/Persistence/DiskTypeFactory.cs b/src/CLU/VhdManagement/Model/Persistence/DiskTypeFactory.cs new file mode 100644 index 000000000000..55b52edc1f12 --- /dev/null +++ b/src/CLU/VhdManagement/Model/Persistence/DiskTypeFactory.cs @@ -0,0 +1,34 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model.Persistence +{ + public class DiskTypeFactory + { + public DiskType Create(uint diskType) + { + switch (diskType) + { + case 0: return DiskType.None; + case 2: return DiskType.Fixed; + case 3: return DiskType.Dynamic; + case 4: return DiskType.Differencing; + default: + throw new VhdParsingException(String.Format("Unsupported format: DiskType is {0}", diskType)); + } + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/Persistence/DynamicDiskBlockFactory.cs b/src/CLU/VhdManagement/Model/Persistence/DynamicDiskBlockFactory.cs new file mode 100644 index 000000000000..760266a596fd --- /dev/null +++ b/src/CLU/VhdManagement/Model/Persistence/DynamicDiskBlockFactory.cs @@ -0,0 +1,77 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model.Persistence +{ + + + public class DynamicDiskBlockFactory : AbstractDiskBlockFactory + { + private BitMapFactory bitMapFactory; + private SectorFactory sectorFactory; + private Block cachedBlock; + + public DynamicDiskBlockFactory(VhdFile vhdFile) : base(vhdFile) + { + this.bitMapFactory = new BitMapFactory(vhdFile); + this.sectorFactory = new SectorFactory(vhdFile, this); + } + + public override Block Create(uint block) + { + if (!vhdFile.BlockAllocationTable.HasData(block)) + { + if (cachedBlock == null || cachedBlock.BlockIndex != block) + { + cachedBlock = new Block(this) + { + BlockIndex = block, + VhdUniqueId = this.vhdFile.Footer.UniqueId, + LogicalRange = IndexRange.FromLength(block * GetBlockSize(), vhdFile.Header.BlockSize), + BitMap = null, + Empty = true + }; + } + return cachedBlock; + } + + if(cachedBlock == null || cachedBlock.BlockIndex != block) + { + cachedBlock = new Block(this) + { + BlockIndex = block, + VhdUniqueId = this.vhdFile.Footer.UniqueId, + LogicalRange = IndexRange.FromLength(block * GetBlockSize(), vhdFile.Header.BlockSize), + BitMap = bitMapFactory.Create(block), + Empty = false + }; + } + return cachedBlock; + } + + public override Sector GetSector(Block block, uint sector) + { + if(block.Empty) + { + return this.sectorFactory.CreateEmptySector(block.BlockIndex, sector); + } + return this.sectorFactory.Create(block, sector); + } + + protected override byte[] DoReadBlockData(Block block) + { + return vhdFile.DataReader.ReadBytes(GetBlockAddress(block.BlockIndex), (int)GetBlockSize()); + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/Persistence/FixedDiskBlockFactory.cs b/src/CLU/VhdManagement/Model/Persistence/FixedDiskBlockFactory.cs new file mode 100644 index 000000000000..36dace9bc6ae --- /dev/null +++ b/src/CLU/VhdManagement/Model/Persistence/FixedDiskBlockFactory.cs @@ -0,0 +1,132 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model.Persistence +{ + public class FixedDiskBlockFactory : IBlockFactory + { + private readonly VhdFile vhdFile; + private SectorFactory sectorFactory; + private Block cachedBlock; + private long blockSize; + private long? extraBlockIndex; + + public FixedDiskBlockFactory(VhdFile vhdFile) : this(vhdFile, VhdConstants.VHD_DEFAULT_BLOCK_SIZE) + { + } + + public FixedDiskBlockFactory(VhdFile vhdFile, long blockSize) + { + this.vhdFile = vhdFile; + this.blockSize = blockSize; + this.BlockCount = CalculateBlockCount(); + this.sectorFactory = new SectorFactory(vhdFile, this); + } + + private int CalculateBlockCount() + { + var count = this.vhdFile.Footer.VirtualSize * 1.0m / this.GetBlockSize(); + if(Math.Floor(count) < Math.Ceiling(count)) + { + extraBlockIndex = (long) Math.Floor(count); + } + return (int)Math.Ceiling(count); + } + + public long BlockCount { get; private set; } + + public Block Create(uint block) + { + if (!this.HasData(block)) + { + if (cachedBlock == null || cachedBlock.BlockIndex != block) + { + IndexRange logicalRange = IndexRange.FromLength(block * GetBlockSize(), this.GetBlockSize()); + if (extraBlockIndex.HasValue && block == extraBlockIndex) + { + long startIndex = block * GetBlockSize(); + long size = this.vhdFile.DataReader.Size - startIndex - VhdConstants.VHD_FOOTER_SIZE; + logicalRange = IndexRange.FromLength(startIndex, size); + } + cachedBlock = new Block(this) + { + BlockIndex = block, + VhdUniqueId = this.vhdFile.Footer.UniqueId, + LogicalRange = logicalRange, + BitMap = null, + Empty = true + }; + } + return cachedBlock; + } + if(cachedBlock == null || cachedBlock.BlockIndex != block) + { + IndexRange logicalRange = IndexRange.FromLength(block * GetBlockSize(), this.GetBlockSize()); + if (extraBlockIndex.HasValue && block == extraBlockIndex) + { + long startIndex = block * GetBlockSize(); + long size = this.vhdFile.DataReader.Size - startIndex - VhdConstants.VHD_FOOTER_SIZE; + logicalRange = IndexRange.FromLength(startIndex, size); + } + cachedBlock = new Block(this) + { + BlockIndex = block, + VhdUniqueId = this.vhdFile.Footer.UniqueId, + LogicalRange = logicalRange, + Empty = false + }; + } + return cachedBlock; + } + + public byte[] ReadBlockData(Block block) + { + long blockAddress = GetBlockAddress(block.BlockIndex); + return vhdFile.DataReader.ReadBytes(blockAddress, (int) block.LogicalRange.Length); + } + + public Sector GetSector(Block block, uint sector) + { + if (block.Empty) + { + return this.sectorFactory.CreateEmptySector(block.BlockIndex, sector); + } + return this.sectorFactory.Create(block, sector); + } + + public IndexRange GetFooterRange() + { + long startIndex = this.vhdFile.DataReader.Size - VhdConstants.VHD_FOOTER_SIZE; + var logicalRange = IndexRange.FromLength(startIndex, VhdConstants.VHD_FOOTER_SIZE); + return logicalRange; + } + + public bool HasData(uint blockIndex) + { + return blockIndex != VhdConstants.VHD_NO_DATA_INT; + } + + public long GetBlockAddress(uint blockIndex) + { + return blockIndex * this.blockSize; + } + + public long GetBlockSize() + { + return this.blockSize; + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/Persistence/IBlockFactory.cs b/src/CLU/VhdManagement/Model/Persistence/IBlockFactory.cs new file mode 100644 index 000000000000..e780a6daf794 --- /dev/null +++ b/src/CLU/VhdManagement/Model/Persistence/IBlockFactory.cs @@ -0,0 +1,32 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model.Persistence +{ + public interface IBlockFactory + { + long BlockCount { get; } + + long GetBlockSize(); + long GetBlockAddress(uint blockIndex); + bool HasData(uint blockIndex); + + Block Create(uint block); + byte[] ReadBlockData(Block block); + + Sector GetSector(Block block, uint sector); + + IndexRange GetFooterRange(); + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/Persistence/SectorFactory.cs b/src/CLU/VhdManagement/Model/Persistence/SectorFactory.cs new file mode 100644 index 000000000000..54b3c7d221a7 --- /dev/null +++ b/src/CLU/VhdManagement/Model/Persistence/SectorFactory.cs @@ -0,0 +1,71 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model.Persistence +{ + public class SectorFactory + { + private readonly VhdFile vhdFile; + private readonly IBlockFactory blockFactory; + + public SectorFactory(VhdFile vhdFile, IBlockFactory blockFactory) + { + this.vhdFile = vhdFile; + this.blockFactory = blockFactory; + } + + public Sector Create(Block blockArg, uint sector) + { + uint block = blockArg.BlockIndex; + long totalSectors = blockArg.SectorCount; + if (sector > totalSectors) + { + string message = String.Format("TotalSectors: {0}, Requested Sector:{1}", totalSectors, sector); + throw new ArgumentOutOfRangeException("sector", message); + } + if (!blockFactory.HasData(block)) + { + return CreateEmptySector(block, sector); + } + + long currentAddress = blockFactory.GetBlockAddress(block); + vhdFile.DataReader.SetPosition(currentAddress + (int)VhdConstants.VHD_SECTOR_LENGTH * sector); + + var result = new Sector + { + BlockIndex = block, + SectorIndex = sector, + GlobalSectorIndex = this.blockFactory.GetBlockSize() * block + sector, + Data = vhdFile.DataReader.ReadBytes((int)VhdConstants.VHD_SECTOR_LENGTH) + }; + return result; + } + + public Sector CreateEmptySector(uint block, uint sector) + { + var buffer = new byte[((int)VhdConstants.VHD_SECTOR_LENGTH)]; + Array.Clear(buffer, 0, buffer.Length); + var emptySector = new Sector + { + BlockIndex = block, + SectorIndex = sector, + GlobalSectorIndex = this.blockFactory.GetBlockSize() * block + sector, + Data = buffer + }; + return emptySector; + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/Persistence/StreamHelper.cs b/src/CLU/VhdManagement/Model/Persistence/StreamHelper.cs new file mode 100644 index 000000000000..2919bb29fa87 --- /dev/null +++ b/src/CLU/VhdManagement/Model/Persistence/StreamHelper.cs @@ -0,0 +1,100 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.IO; +using Microsoft.WindowsAzure.Commands.Tools.Common.General; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model.Persistence +{ + public class StreamHelper + { + public static IAsyncResult BeginReadBytes(Stream stream, long offset, int length, AsyncCallback callback, object state) + { + stream.Seek(offset, SeekOrigin.Begin); + return AsyncMachine.BeginAsyncMachine(ReadBytesAsync, stream, length, callback, state); + } + + public static IAsyncResult BeginReadBytes(Stream stream, long offset, int length, SeekOrigin origin, AsyncCallback callback, object state) + { + stream.Seek(-offset, SeekOrigin.End); + return AsyncMachine.BeginAsyncMachine(ReadBytesAsync, stream, length, callback, state); + } + + public static byte[] EndReadBytes(IAsyncResult result) + { + return AsyncMachine.EndAsyncMachine(result); + } + + public static IEnumerable ReadBytesAsync(AsyncMachine machine, Stream stream, int length) + { + var attributeHelper = new AttributeHelper(); + var buffer = new byte[length]; + + int readCount = 0; + int remaining = length; + while (remaining > 0) + { + // TODO: CLU + yield return CompletionPort.SingleOperation; + var currentRead = stream.Read(buffer, readCount, remaining); + /* + stream.BeginRead(buffer, readCount, remaining, machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + var currentRead = stream.EndRead(machine.CompletionResult); + */ + if (currentRead == 0) + { + break; + } + readCount += currentRead; + remaining -= currentRead; + } + machine.ParameterValue = buffer; + yield break; + } + + public static byte[] ReadBytes(Stream stream, long offset, int length) + { + stream.Seek(offset, SeekOrigin.Begin); + return ReadBytes(stream, length); + } + + public static byte[] ReadBytes(Stream stream, long offset, int length, SeekOrigin origin) + { + stream.Seek(-offset, origin); + return ReadBytes(stream, length); + } + + private static byte[] ReadBytes(Stream stream, int length) + { + var buffer = new byte[length]; + int readCount = 0; + int remaining = length; + while (remaining > 0) + { + var currentRead = stream.Read(buffer, readCount, remaining); + if (currentRead == 0) + { + break; + } + readCount += currentRead; + remaining -= currentRead; + } + + return buffer; + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/Persistence/VhdConstants.cs b/src/CLU/VhdManagement/Model/Persistence/VhdConstants.cs new file mode 100644 index 000000000000..a2f6c18eddcc --- /dev/null +++ b/src/CLU/VhdManagement/Model/Persistence/VhdConstants.cs @@ -0,0 +1,28 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model.Persistence +{ + public class VhdConstants + { + public const long VHD_DEFAULT_BLOCK_SIZE = 512 * 1024; + public const long VHD_NO_DATA_LONG = ~0L; + public const uint VHD_NO_DATA_INT = 0xFFFFFFFF; + + public const long VHD_PAGE_SIZE = 512; + public const long VHD_FOOTER_SIZE = 512; + public const long VHD_SECTOR_LENGTH = 512; + public const long VHD_FOOTER_OFFSET_CHECKSUM = 64; + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/Persistence/VhdDataReader.cs b/src/CLU/VhdManagement/Model/Persistence/VhdDataReader.cs new file mode 100644 index 000000000000..a32eef5d707b --- /dev/null +++ b/src/CLU/VhdManagement/Model/Persistence/VhdDataReader.cs @@ -0,0 +1,302 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Text; +using Microsoft.WindowsAzure.Commands.Tools.Common.General; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model.Persistence +{ + public class VhdDataReader + { + private readonly BinaryReader reader; + private byte[] m_buffer; + + public VhdDataReader(BinaryReader reader) + { + this.reader = reader; + this.m_buffer = new byte[16]; + } + + public long Size + { + get { return this.reader.BaseStream.Length; } + } + + public bool ReadBoolean(long offset) + { + this.SetPosition(offset); + return this.reader.ReadBoolean(); + } + + public IAsyncResult BeginReadBoolean(long offset, AsyncCallback callback, object state) + { + this.SetPosition(offset); + return AsyncMachine.BeginAsyncMachine(FillBuffer, 1, callback, state); + } + + public bool EndReadBoolean(IAsyncResult result) + { + AsyncMachine.EndAsyncMachine(result); + return (m_buffer[0] != 0); + } + + IEnumerable FillBuffer(AsyncMachine machine, int numBytes) + { + if (m_buffer != null && (numBytes < 0 || numBytes > m_buffer.Length)) + { + throw new ArgumentOutOfRangeException("numBytes", String.Format("Expected (0-16) however found: {0}", numBytes)); + } + int bytesRead = 0; + int n = 0; + + // Need to find a good threshold for calling ReadByte() repeatedly + // vs. calling Read(byte[], int, int) for both buffered & unbuffered + // streams. + if (numBytes == 1) + { + // TODO: CLU + yield return CompletionPort.SingleOperation; + n = this.reader.BaseStream.Read(m_buffer, 0, numBytes); + /* + this.reader.BaseStream.BeginRead(m_buffer, 0, numBytes, machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + n = this.reader.BaseStream.EndRead(machine.CompletionResult); + */ + if (n == -1) + { + throw new EndOfStreamException(); + } + m_buffer[0] = (byte)n; + } + + do + { + // TODO: CLU + yield return CompletionPort.SingleOperation; + n = this.reader.BaseStream.Read(m_buffer, bytesRead, numBytes); + /* + this.reader.BaseStream.BeginRead(m_buffer, bytesRead, numBytes - bytesRead, machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + n = this.reader.BaseStream.EndRead(machine.CompletionResult); + */ + + if (n == 0) + { + throw new EndOfStreamException(); + } + bytesRead += n; + } while (bytesRead < numBytes); + } + + public short ReadInt16(long offset) + { + this.SetPosition(offset); + return IPAddress.NetworkToHostOrder((short)this.reader.ReadUInt16()); + } + + public IAsyncResult BeginReadInt16(long offset, AsyncCallback callback, object state) + { + this.SetPosition(offset); + return AsyncMachine.BeginAsyncMachine(FillBuffer, 2, callback, state); + } + + public short EndReadInt16(IAsyncResult result) + { + AsyncMachine.EndAsyncMachine(result); + short value = (short) (m_buffer[0] | m_buffer[1] << 8); + return IPAddress.NetworkToHostOrder(value); + } + + public uint ReadUInt32(long offset) + { + this.SetPosition(offset); + return (uint)IPAddress.NetworkToHostOrder((int)this.reader.ReadUInt32()); + } + + public IAsyncResult BeginReadUInt32(long offset, AsyncCallback callback, object state) + { + this.SetPosition(offset); + return AsyncMachine.BeginAsyncMachine(FillBuffer, 4, callback, state); + } + + public uint EndReadUInt32(IAsyncResult result) + { + AsyncMachine.EndAsyncMachine(result); + var value = (m_buffer[0] | m_buffer[1] << 8 | m_buffer[2] << 16 | m_buffer[3] << 24); + return (uint)IPAddress.NetworkToHostOrder(value); + } + + public uint ReadUInt32() + { + return (uint)IPAddress.NetworkToHostOrder((int)this.reader.ReadUInt32()); + } + + public IAsyncResult BeginReadUInt32(AsyncCallback callback, object state) + { + return AsyncMachine.BeginAsyncMachine(FillBuffer, 4, callback, state); + } + + public ulong ReadUInt64(long offset) + { + this.SetPosition(offset); + var value = (long)this.reader.ReadUInt64(); + return (ulong)IPAddress.NetworkToHostOrder(value); + } + + public IAsyncResult BeginReadUInt64(long offset, AsyncCallback callback, object state) + { + this.SetPosition(offset); + return AsyncMachine.BeginAsyncMachine(FillBuffer, 8, callback, state); + } + + public ulong EndReadUInt64(IAsyncResult result) + { + AsyncMachine.EndAsyncMachine(result); + uint lo = (uint)(m_buffer[0] | m_buffer[1] << 8 | + m_buffer[2] << 16 | m_buffer[3] << 24); + uint hi = (uint)(m_buffer[4] | m_buffer[5] << 8 | + m_buffer[6] << 16 | m_buffer[7] << 24); + ulong value = ((ulong) hi) << 32 | lo; + return (ulong)IPAddress.NetworkToHostOrder((long)value); + } + + public byte[] ReadBytes(long offset, int count) + { + this.SetPosition(offset); + return this.reader.ReadBytes(count); + } + + public IAsyncResult BeginReadBytes(long offset, int count, AsyncCallback callback, object state) + { + this.SetPosition(offset); + return AsyncMachine.BeginAsyncMachine(ReadBytesAsync, offset, count, callback, state); + } + + public byte[] EndReadBytes(IAsyncResult result) + { + return AsyncMachine.EndAsyncMachine(result); + } + + private IEnumerable ReadBytesAsync(AsyncMachine machine, long offset, int count) + { + StreamHelper.BeginReadBytes(this.reader.BaseStream, offset, count, machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + byte[] values = StreamHelper.EndReadBytes(machine.CompletionResult); + machine.ParameterValue = values; + } + + public byte[] ReadBytes(int count) + { + return this.reader.ReadBytes(count); + } + + public IAsyncResult BeginReadBytes(int count, AsyncCallback callback, object state) + { + return AsyncMachine.BeginAsyncMachine(ReadBytesAsync, this.reader.BaseStream.Position, count, callback, state); + } + + public string ReadString(int count) + { + return Encoding.ASCII.GetString(this.reader.ReadBytes(count)); + } + + public IAsyncResult BeginReadString(int count, AsyncCallback callback, object state) + { + return AsyncMachine.BeginAsyncMachine(ReadStringAsync, this.reader.BaseStream.Position, count, callback, state); + } + + public string EndReadString(IAsyncResult result) + { + return AsyncMachine.EndAsyncMachine(result); + } + + private IEnumerable ReadStringAsync(AsyncMachine machine, long offset, int count) + { + BeginReadBytes(offset, count, machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + byte[] values = EndReadBytes(machine.CompletionResult); + machine.ParameterValue = Encoding.ASCII.GetString(values); + } + + public byte ReadByte(long offset) + { + this.SetPosition(offset); + return this.reader.ReadByte(); + } + + public IAsyncResult BeginReadByte(long offset, AsyncCallback callback, object state) + { + return BeginReadBytes(offset, 1, callback, state); + } + + public byte EndReadByte(IAsyncResult result) + { + return EndReadBytes(result)[0]; + } + + public Guid ReadGuid(long offset) + { + return new Guid(this.ReadBytes(offset, 16)); + } + + public IAsyncResult BeginReadGuid(long offset, AsyncCallback callback, object state) + { + return BeginReadBytes(offset, 16, callback, state); + } + + public Guid EndReadGuid(IAsyncResult result) + { + byte[] guidValue = EndReadBytes(result); + return new Guid(guidValue); + } + + public DateTime ReadDateTime(long offset) + { + var timeStamp = new VhdTimeStamp(this.ReadUInt32(offset)); + return timeStamp.ToDateTime(); + } + + public IAsyncResult BeginReadDateTime(long offset, AsyncCallback callback, object state) + { + return BeginReadUInt32(offset, callback, state); + } + + public DateTime EndReadDateTime(IAsyncResult result) + { + uint value = EndReadUInt32(result); + var timeStamp = new VhdTimeStamp(value); + return timeStamp.ToDateTime(); + } + + public DateTime ReadDateTime() + { + var timeStamp = new VhdTimeStamp(this.ReadUInt32()); + return timeStamp.ToDateTime(); + } + + public IAsyncResult BeginReadDateTime(AsyncCallback callback, object state) + { + return BeginReadUInt32(callback, state); + } + + public void SetPosition(long batOffset) + { + this.reader.BaseStream.Seek(batOffset, SeekOrigin.Begin); + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/Persistence/VhdDataWriter.cs b/src/CLU/VhdManagement/Model/Persistence/VhdDataWriter.cs new file mode 100644 index 000000000000..97f1b87da086 --- /dev/null +++ b/src/CLU/VhdManagement/Model/Persistence/VhdDataWriter.cs @@ -0,0 +1,112 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.IO; +using System.Net; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model.Persistence +{ + public class VhdDataWriter + { + private readonly BinaryWriter writer; + + public VhdDataWriter(BinaryWriter writer) + { + this.writer = writer; + } + + public long Size + { + get { return this.writer.BaseStream.Length; } + } + + public void WriteBoolean(long offset, bool value) + { + this.SetPosition(offset); + this.writer.Write(value); + } + + public void WriteInt(long offset, int value) + { + this.SetPosition(offset); + this.writer.Write((uint)IPAddress.HostToNetworkOrder(value)); + } + + public void WriteInt16(long offset, Int16 value) + { + this.SetPosition(offset); + this.writer.Write(IPAddress.HostToNetworkOrder((short)value)); + } + + public void WriteInt16(Int16 value) + { + this.writer.Write(IPAddress.HostToNetworkOrder((short)value)); + } + + public void WriteUInt(long offset, uint value) + { + this.SetPosition(offset); + this.writer.Write((uint)IPAddress.HostToNetworkOrder((int)value)); + } + + public void WriteUInt(uint value) + { + this.writer.Write((uint)IPAddress.HostToNetworkOrder((int)value)); + } + + public void WriteLong(long offset, long value) + { + this.SetPosition(offset); + var result = (ulong)IPAddress.HostToNetworkOrder(value); + this.writer.Write(result); + } + + public void WriteBytes(long offset, byte[] value) + { + this.SetPosition(offset); + this.writer.Write(value); + } + + public void WriteByte(long offset, byte value) + { + this.SetPosition(offset); + this.writer.Write(value); + } + + public void WriteByte(byte value) + { + this.writer.Write(value); + } + + public void WriteGuid(long offset, Guid value) + { + this.SetPosition(offset); + this.writer.Write(value.ToByteArray()); + } + + public void WriteTimeStamp(long offset, DateTime value) + { + this.SetPosition(offset); + var timeStamp = new VhdTimeStamp(value); + uint result = (uint)IPAddress.HostToNetworkOrder((int)timeStamp.TotalSeconds); + this.writer.Write(result); + } + + public void SetPosition(long batOffset) + { + this.writer.BaseStream.Seek(batOffset, SeekOrigin.Begin); + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/Persistence/VhdFileFactory.cs b/src/CLU/VhdManagement/Model/Persistence/VhdFileFactory.cs new file mode 100644 index 000000000000..43fb630b1b9d --- /dev/null +++ b/src/CLU/VhdManagement/Model/Persistence/VhdFileFactory.cs @@ -0,0 +1,212 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using Microsoft.WindowsAzure.Commands.Tools.Common.General; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model.Persistence +{ + public class VhdFileFactory + { + public VhdFile Create(string path) + { + var streamSource = new StreamSource + { + Stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 1024), + VhdDirectory = Path.GetDirectoryName(path), + DisposeOnException = true + }; + return Create(streamSource); + } + + public VhdFile Create(Stream stream) + { + return Create(new StreamSource { Stream = stream }); + } + + private VhdFile Create(StreamSource streamSource) + { + var disposer = new Action(() => { if (streamSource.DisposeOnException) streamSource.Stream.Dispose(); }); + bool throwing = false; + try + { + var reader = new BinaryReader(streamSource.Stream, Encoding.Unicode); + var dataReader = new VhdDataReader(reader); + var footer = new VhdFooterFactory(dataReader).CreateFooter(); + + VhdHeader header = null; + BlockAllocationTable blockAllocationTable = null; + VhdFile parent = null; + if (footer.DiskType != DiskType.Fixed) + { + header = new VhdHeaderFactory(dataReader, footer).CreateHeader(); + blockAllocationTable = new BlockAllocationTableFactory(dataReader, header).Create(); + if (footer.DiskType == DiskType.Differencing) + { + var parentPath = streamSource.VhdDirectory == null ? header.ParentPath : Path.Combine(streamSource.VhdDirectory, header.GetRelativeParentPath()); + parent = Create(parentPath); + } + } + return new VhdFile(footer, header, blockAllocationTable, parent, streamSource.Stream); + } + catch (Exception e) + { + throwing = true; + throw new VhdParsingException("unsupported format", e); + } + finally + { + if(throwing) + { + disposer(); + } + } + } + + private T TryCatch(Func method, IAsyncResult result) + { + try + { + return method(result); + } + catch (EndOfStreamException e) + { + throw new VhdParsingException("unsupported format", e); + } + } + + private T TryCatch(Func method, Action disposer, IAsyncResult result) + { + bool throwing = true; + T methodResult = default(T); + try + { + methodResult = method(result); + throwing = false; + } + catch (EndOfStreamException e) + { + throw new VhdParsingException("unsupported format", e); + } + finally + { + if(throwing) + { + disposer(); + } + } + return methodResult; + } + + private T TryCatch(Func method, Action disposer) + { + bool throwing = true; + T methodResult = default(T); + try + { + methodResult = method(); + throwing = false; + } + catch (EndOfStreamException e) + { + throw new VhdParsingException("unsupported format", e); + } + finally + { + if(throwing) + { + disposer(); + } + } + return methodResult; + } + + public IAsyncResult BeginCreate(string path, AsyncCallback callback, object state) + { + var streamSource = new StreamSource + { + Stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 1024), + VhdDirectory = Path.GetDirectoryName(path), + DisposeOnException = true + }; + return AsyncMachine.BeginAsyncMachine(this.CreateAsync, streamSource, callback, state); + } + + public IAsyncResult BeginCreate(Stream stream, AsyncCallback callback, object state) + { + var streamSource = new StreamSource { Stream = stream}; + return AsyncMachine.BeginAsyncMachine(this.CreateAsync, streamSource, callback, state); + } + + public VhdFile EndCreate(IAsyncResult result) + { + return AsyncMachine.EndAsyncMachine(result); + } + + class StreamSource + { + public Stream Stream { get; set; } + public string VhdDirectory { get; set; } + public bool DisposeOnException { get; set; } + + public StreamSource() + { + this.DisposeOnException = false; + } + } + + private IEnumerable CreateAsync(AsyncMachine machine, StreamSource streamSource) + { + var disposer = new Action(() => { if (streamSource.DisposeOnException) streamSource.Stream.Dispose(); }); + + var reader = TryCatch(() => new BinaryReader(streamSource.Stream, Encoding.Unicode), disposer); + var dataReader = TryCatch(() => new VhdDataReader(reader), disposer); + var footerFactory = TryCatch(() => new VhdFooterFactory(dataReader), disposer); + + footerFactory.BeginCreateFooter(machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + var footer = TryCatch(footerFactory.EndCreateFooter, disposer, machine.CompletionResult); + + VhdHeader header = null; + BlockAllocationTable blockAllocationTable = null; + VhdFile parent = null; + if (footer.DiskType != DiskType.Fixed) + { + var headerFactory = new VhdHeaderFactory(dataReader, footer); + + headerFactory.BeginCreateHeader(machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + header = TryCatch(headerFactory.EndCreateHeader, disposer, machine.CompletionResult); + + var tableFactory = new BlockAllocationTableFactory(dataReader, header); + tableFactory.BeginCreate(machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + blockAllocationTable = TryCatch(tableFactory.EndCreate, disposer, machine.CompletionResult); + + if (footer.DiskType == DiskType.Differencing) + { + var parentPath = streamSource.VhdDirectory == null ? header.ParentPath : Path.Combine(streamSource.VhdDirectory, header.GetRelativeParentPath()); + + BeginCreate(parentPath, machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + parent = TryCatch(EndCreate, disposer, machine.CompletionResult); + } + } + machine.ParameterValue = new VhdFile(footer, header, blockAllocationTable, parent, streamSource.Stream); + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/Persistence/VhdFooterFactory.cs b/src/CLU/VhdManagement/Model/Persistence/VhdFooterFactory.cs new file mode 100644 index 000000000000..64d51eb3a53e --- /dev/null +++ b/src/CLU/VhdManagement/Model/Persistence/VhdFooterFactory.cs @@ -0,0 +1,532 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using Microsoft.WindowsAzure.Commands.Tools.Common.General; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model.Persistence +{ + public class VhdFooterFactory + { + public const string WindowsAzureCreatorApplicationName = "wa"; + + public static VhdFooter CreateFixedDiskFooter(long virtualSize) + { + var helper = new AttributeHelper(); + var footer = new VhdFooter(); + var reservedAttribute = helper.GetAttribute(() => footer.Reserved); + + footer.Cookie = VhdCookie.CreateFooterCookie(); + footer.Features = VhdFeature.Reserved; + footer.FileFormatVersion = VhdFileFormatVersion.DefaultFileFormatVersion; + footer.HeaderOffset = VhdConstants.VHD_NO_DATA_LONG; + footer.TimeStamp = DateTime.UtcNow; + footer.CreatorApplication = WindowsAzureCreatorApplicationName; + footer.CreatorVersion = VhdCreatorVersion.CSUP2011; + footer.CreatorHostOsType = HostOsType.Windows; + footer.PhsyicalSize = virtualSize; + footer.VirtualSize = virtualSize; + footer.DiskGeometry = DiskGeometry.CreateFromVirtualSize(virtualSize); + footer.DiskType = DiskType.Fixed; + footer.UniqueId = Guid.NewGuid(); + footer.SavedState = false; + footer.Reserved = new byte[reservedAttribute.Size]; + + var footerSerializer = new VhdFooterSerializer(footer); + var byteArray = footerSerializer.ToByteArray(); + + using(var memoryStream = new MemoryStream(byteArray)) + { + var binaryReader = new BinaryReader(memoryStream); + var vhdDataReader = new VhdDataReader(binaryReader); + var footerFactory = new VhdFooterFactory(vhdDataReader); + var vhdFooter = footerFactory.CreateFooter(); + return vhdFooter; + } + } + + private readonly VhdDataReader dataReader; + private readonly DiskTypeFactory diskTypeFactory; + + public VhdFooterFactory(VhdDataReader dataReader) + { + this.dataReader = dataReader; + this.diskTypeFactory = new DiskTypeFactory(); + } + + public VhdFooter CreateFooter() + { + try + { + ValidateVhdSize(); + var attributeHelper = new AttributeHelper(); + var footer = new VhdFooter(); + footer.Cookie = ReadVhdCookie(attributeHelper.GetAttribute(() => footer.Cookie)); + footer.Features = ReadFeatures(attributeHelper.GetAttribute(() => footer.Features)); + footer.FileFormatVersion = ReadVhdFileFormatVersion(attributeHelper.GetAttribute(() => footer.FileFormatVersion)); + footer.HeaderOffset = ReadHeaderOffset(attributeHelper.GetAttribute(() => footer.HeaderOffset)); + footer.TimeStamp = ReadTimeStamp(attributeHelper.GetAttribute(() => footer.TimeStamp)); + footer.CreatorApplication = ReadCreatorApplication(attributeHelper.GetAttribute(() => footer.CreatorApplication)); + footer.CreatorVersion = ReadCreatorVersion(attributeHelper.GetAttribute(() => footer.CreatorVersion)); + footer.CreatorHostOsType = ReadCreatorHostOsType(attributeHelper.GetAttribute(() => footer.CreatorHostOsType)); + footer.PhsyicalSize = ReadPhysicalSize(attributeHelper.GetAttribute(() => footer.PhsyicalSize)); + footer.VirtualSize = ReadVirtualSize(attributeHelper.GetAttribute(() => footer.VirtualSize)); + footer.DiskGeometry = ReadDiskGeometry(attributeHelper.GetAttribute(() => footer.DiskGeometry)); + footer.DiskType = ReadDiskType(attributeHelper.GetAttribute(() => footer.DiskType)); + footer.CheckSum = ReadCheckSum(attributeHelper.GetAttribute(() => footer.CheckSum)); + footer.UniqueId = ReadUniqueId(attributeHelper.GetAttribute(() => footer.UniqueId)); + footer.SavedState = ReadSavedState(attributeHelper.GetAttribute(() => footer.SavedState)); + footer.Reserved = ReadReserved(attributeHelper.GetAttribute(() => footer.Reserved)); + footer.RawData = ReadWholeFooter(attributeHelper.GetAttribute(() => footer.RawData)); + return footer; + } + catch (EndOfStreamException e) + { + throw new VhdParsingException("unsupported format", e); + } + } + + private T TryCatch(Func method, IAsyncResult result) + { + try + { + return method(result); + } + catch (EndOfStreamException e) + { + throw new VhdParsingException("unsupported format", e); + } + } + + public IAsyncResult BeginCreateFooter(AsyncCallback callback, object state) + { + return AsyncMachine.BeginAsyncMachine(CreateFooterAsync, callback, state); + } + + public VhdFooter EndCreateFooter(IAsyncResult result) + { + return AsyncMachine.EndAsyncMachine(result); + } + + private IEnumerable CreateFooterAsync(AsyncMachine machine) + { + ValidateVhdSize(); + var attributeHelper = new AttributeHelper(); + var footer = new VhdFooter(); + + BeginReadVhdCookie(attributeHelper.GetAttribute(() => footer.Cookie), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + footer.Cookie = TryCatch(EndReadVhdCookie, machine.CompletionResult); + + BeginReadFeatures(attributeHelper.GetAttribute(() => footer.Features), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + footer.Features = TryCatch(EndReadFeatures, machine.CompletionResult); + + BeginReadVhdFileFormatVersion(attributeHelper.GetAttribute(() => footer.FileFormatVersion), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + footer.FileFormatVersion = TryCatch(EndReadVhdFileFormatVersion, machine.CompletionResult); + + BeginReadHeaderOffset(attributeHelper.GetAttribute(() => footer.HeaderOffset), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + footer.HeaderOffset = TryCatch(EndReadHeaderOffset, machine.CompletionResult); + + BeginReadTimeStamp(attributeHelper.GetAttribute(() => footer.TimeStamp), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + footer.TimeStamp = TryCatch(EndReadTimeStamp, machine.CompletionResult); + + BeginReadCreatorApplication(attributeHelper.GetAttribute(() => footer.CreatorApplication), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + footer.CreatorApplication = TryCatch(EndReadCreatorApplication, machine.CompletionResult); + + BeginReadCreatorVersion(attributeHelper.GetAttribute(() => footer.CreatorVersion), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + footer.CreatorVersion = TryCatch(EndReadCreatorVersion, machine.CompletionResult); + + BeginReadCreatorHostOsType(attributeHelper.GetAttribute(() => footer.CreatorHostOsType), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + footer.CreatorHostOsType = TryCatch(EndReadCreatorHostOsType, machine.CompletionResult); + + BeginReadPhysicalSize(attributeHelper.GetAttribute(() => footer.PhsyicalSize), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + footer.PhsyicalSize = TryCatch(EndReadPhysicalSize, machine.CompletionResult); + + BeginReadVirtualSize(attributeHelper.GetAttribute(() => footer.VirtualSize), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + footer.VirtualSize = TryCatch(EndReadVirtualSize, machine.CompletionResult); + + BeginReadDiskGeometry(attributeHelper.GetAttribute(() => footer.DiskGeometry), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + footer.DiskGeometry = TryCatch(EndReadDiskGeometry, machine.CompletionResult); + + BeginReadDiskType(attributeHelper.GetAttribute(() => footer.DiskType), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + footer.DiskType = TryCatch(EndReadDiskType, machine.CompletionResult); + + BeginReadCheckSum(attributeHelper.GetAttribute(() => footer.CheckSum), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + footer.CheckSum = TryCatch(EndReadCheckSum, machine.CompletionResult); + + BeginReadUniqueId(attributeHelper.GetAttribute(() => footer.UniqueId), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + footer.UniqueId = TryCatch(EndReadUniqueId, machine.CompletionResult); + + BeginReadSavedState(attributeHelper.GetAttribute(() => footer.SavedState), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + footer.SavedState = TryCatch(EndReadSavedState, machine.CompletionResult); + + BeginReadReserved(attributeHelper.GetAttribute(() => footer.Reserved), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + footer.Reserved = TryCatch(EndReadReserved, machine.CompletionResult); + + BeginReadWholeFooter(attributeHelper.GetAttribute(() => footer.RawData), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + footer.RawData = TryCatch(EndReadWholeFooter, machine.CompletionResult); + + machine.ParameterValue = footer; + } + + private long ReadPhysicalSize(VhdPropertyAttribute attribute) + { + return (long) dataReader.ReadUInt64(this.GetFooterOffset() + attribute.Offset); + } + + private IAsyncResult BeginReadPhysicalSize(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadUInt64(this.GetFooterOffset() + attribute.Offset, callback, state); + } + + private long EndReadPhysicalSize(IAsyncResult result) + { + var value = dataReader.EndReadUInt64(result); + return (long) value; + } + + private byte[] ReadWholeFooter(VhdPropertyAttribute attribute) + { + return dataReader.ReadBytes(GetFooterOffset() + attribute.Offset, attribute.Size); + } + + private IAsyncResult BeginReadWholeFooter(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadBytes(this.GetFooterOffset() + attribute.Offset, attribute.Size, callback, state); + } + + private byte[] EndReadWholeFooter(IAsyncResult result) + { + var value = dataReader.EndReadBytes(result); + return (byte[]) value; + } + + private byte[] ReadReserved(VhdPropertyAttribute attribute) + { + return dataReader.ReadBytes(GetFooterOffset() + attribute.Offset, attribute.Size); + } + + private IAsyncResult BeginReadReserved(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadBytes(this.GetFooterOffset() + attribute.Offset, attribute.Size, callback, state); + } + + private byte[] EndReadReserved(IAsyncResult result) + { + var value = dataReader.EndReadBytes(result); + return (byte[]) value; + } + + private bool ReadSavedState(VhdPropertyAttribute attribute) + { + return dataReader.ReadBoolean(GetFooterOffset() + attribute.Offset); + } + + private IAsyncResult BeginReadSavedState(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadBoolean(this.GetFooterOffset() + attribute.Offset, callback, state); + } + + private bool EndReadSavedState(IAsyncResult result) + { + var value = dataReader.EndReadBoolean(result); + return (bool) value; + } + + private uint ReadCheckSum(VhdPropertyAttribute attribute) + { + return dataReader.ReadUInt32(GetFooterOffset() + attribute.Offset); + } + + private IAsyncResult BeginReadCheckSum(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadUInt32(this.GetFooterOffset() + attribute.Offset, callback, state); + } + + private uint EndReadCheckSum(IAsyncResult result) + { + var value = dataReader.EndReadUInt32(result); + return (uint) value; + } + + private DiskGeometry ReadDiskGeometry(VhdPropertyAttribute attribute) + { + long offset = GetFooterOffset() + attribute.Offset; + + var attributeHelper = new AttributeHelper(); + var diskGeometry = new DiskGeometry(); + diskGeometry.Cylinder = dataReader.ReadInt16(offset + attributeHelper.GetAttribute(()=>diskGeometry.Cylinder).Offset); + diskGeometry.Heads = dataReader.ReadByte(offset + attributeHelper.GetAttribute(() => diskGeometry.Heads).Offset); + diskGeometry.Sectors = dataReader.ReadByte(offset + attributeHelper.GetAttribute(() => diskGeometry.Sectors).Offset); + return diskGeometry; + } + + private IAsyncResult BeginReadDiskGeometry(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return AsyncMachine.BeginAsyncMachine(ReadDiskGeometryAsync, attribute, callback, state); + } + + private DiskGeometry EndReadDiskGeometry(IAsyncResult result) + { + return AsyncMachine.EndAsyncMachine(result); + } + + private IEnumerable ReadDiskGeometryAsync(AsyncMachine machine, VhdPropertyAttribute attribute) + { + long offset = GetFooterOffset() + attribute.Offset; + + var attributeHelper = new AttributeHelper(); + var diskGeometry = new DiskGeometry(); + dataReader.BeginReadInt16(offset + attributeHelper.GetAttribute(() => diskGeometry.Cylinder).Offset, machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + diskGeometry.Cylinder = dataReader.EndReadInt16(machine.CompletionResult); + + dataReader.BeginReadByte(offset + attributeHelper.GetAttribute(() => diskGeometry.Heads).Offset, machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + diskGeometry.Heads = dataReader.EndReadByte(machine.CompletionResult); + + dataReader.BeginReadByte(offset + attributeHelper.GetAttribute(() => diskGeometry.Sectors).Offset, machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + diskGeometry.Sectors = dataReader.EndReadByte(machine.CompletionResult); + + machine.ParameterValue = diskGeometry; + } + + private HostOsType ReadCreatorHostOsType(VhdPropertyAttribute attribute) + { + var hostOs = dataReader.ReadUInt32(GetFooterOffset() + attribute.Offset); + return (HostOsType)hostOs; + } + + private IAsyncResult BeginReadCreatorHostOsType(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadUInt32(this.GetFooterOffset() + attribute.Offset, callback, state); + } + + private HostOsType EndReadCreatorHostOsType(IAsyncResult result) + { + var version = dataReader.EndReadUInt32(result); + return (HostOsType)version; + } + + private VhdCreatorVersion ReadCreatorVersion(VhdPropertyAttribute attribute) + { + var version = dataReader.ReadUInt32(GetFooterOffset() + attribute.Offset); + return new VhdCreatorVersion(version); + } + + private IAsyncResult BeginReadCreatorVersion(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadUInt32(this.GetFooterOffset() + attribute.Offset, callback, state); + } + + private VhdCreatorVersion EndReadCreatorVersion(IAsyncResult result) + { + var version = dataReader.EndReadUInt32(result); + return new VhdCreatorVersion(version); + } + + private string ReadCreatorApplication(VhdPropertyAttribute attribute) + { + var creatorApplication = dataReader.ReadBytes(GetFooterOffset() + attribute.Offset, attribute.Size); + return Encoding.ASCII.GetString(creatorApplication); + } + + private IAsyncResult BeginReadCreatorApplication(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadBytes(this.GetFooterOffset() + attribute.Offset, attribute.Size, callback, state); + } + + private string EndReadCreatorApplication(IAsyncResult result) + { + var creatorApplication = dataReader.EndReadBytes(result); + return Encoding.ASCII.GetString(creatorApplication); + } + + private VhdFeature ReadFeatures(VhdPropertyAttribute attribute) + { + return (VhdFeature)dataReader.ReadUInt32(GetFooterOffset() + attribute.Offset); + } + + private IAsyncResult BeginReadFeatures(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadUInt32(this.GetFooterOffset() + attribute.Offset, callback, state); + } + + private VhdFeature EndReadFeatures(IAsyncResult result) + { + return (VhdFeature) dataReader.EndReadUInt32(result); + } + + private Guid ReadUniqueId(VhdPropertyAttribute attribute) + { + return dataReader.ReadGuid(GetFooterOffset() + attribute.Offset); + } + + private IAsyncResult BeginReadUniqueId(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadGuid(this.GetFooterOffset() + attribute.Offset, callback, state); + } + + private Guid EndReadUniqueId(IAsyncResult result) + { + return dataReader.EndReadGuid(result); + } + + private DateTime ReadTimeStamp(VhdPropertyAttribute attribute) + { + return dataReader.ReadDateTime(GetFooterOffset() + attribute.Offset); + } + + private IAsyncResult BeginReadTimeStamp(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadDateTime(this.GetFooterOffset() + attribute.Offset, callback, state); + } + + private DateTime EndReadTimeStamp(IAsyncResult result) + { + return dataReader.EndReadDateTime(result); + } + + private long ReadHeaderOffset(VhdPropertyAttribute attribute) + { + return (long) dataReader.ReadUInt64(GetFooterOffset() + attribute.Offset); + } + + private IAsyncResult BeginReadHeaderOffset(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadUInt64(this.GetFooterOffset() + attribute.Offset, callback, state); + } + + private long EndReadHeaderOffset(IAsyncResult result) + { + return (long)dataReader.EndReadUInt64(result); + } + + private DiskType ReadDiskType(VhdPropertyAttribute attribute) + { + var readDiskType = dataReader.ReadUInt32(GetFooterOffset() + attribute.Offset); + return diskTypeFactory.Create(readDiskType); + } + + private IAsyncResult BeginReadDiskType(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadUInt32(this.GetFooterOffset() + attribute.Offset, callback, state); + } + + private DiskType EndReadDiskType(IAsyncResult result) + { + var readDiskType = dataReader.EndReadUInt32(result); + return diskTypeFactory.Create(readDiskType); + } + + private long ReadVirtualSize(VhdPropertyAttribute attribute) + { + return (long) dataReader.ReadUInt64(this.GetFooterOffset() + attribute.Offset); + } + + private IAsyncResult BeginReadVirtualSize(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadUInt64(this.GetFooterOffset() + attribute.Offset, callback, state); + } + + private long EndReadVirtualSize(IAsyncResult result) + { + return (long) dataReader.EndReadUInt64(result); + } + + private void ValidateVhdSize() + { + // all VHDs are a multiple of 512 bytes. Note that Azure page blobs must + // be a multiple of 512 bytes also. + var streamLength = dataReader.Size; + if (streamLength == 0 || streamLength < VhdConstants.VHD_FOOTER_SIZE || streamLength % VhdConstants.VHD_PAGE_SIZE != 0) + throw new VhdParsingException(String.Format("Invalid file Size: {0}", streamLength)); + } + + private VhdFileFormatVersion ReadVhdFileFormatVersion(VhdPropertyAttribute attribute) + { + var version = dataReader.ReadUInt32(this.GetFooterOffset() + attribute.Offset); + return CreateVhdFileFormatVersion(version); + } + + private VhdFileFormatVersion CreateVhdFileFormatVersion(uint version) + { + var formatVersion = new VhdFileFormatVersion(version); + if (!formatVersion.IsSupported()) + throw new VhdParsingException(String.Format("Invalid file format version: {0}", formatVersion.Data)); + return formatVersion; + } + + private IAsyncResult BeginReadVhdFileFormatVersion(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadUInt32(this.GetFooterOffset() + attribute.Offset, callback, state); + } + + private VhdFileFormatVersion EndReadVhdFileFormatVersion(IAsyncResult result) + { + uint value = dataReader.EndReadUInt32(result); + return CreateVhdFileFormatVersion(value); + } + + private VhdCookie ReadVhdCookie(VhdPropertyAttribute attribute) + { + byte[] value = dataReader.ReadBytes(this.GetFooterOffset() + attribute.Offset, attribute.Size); + return CreateVhdCookie(value); + } + + private VhdCookie CreateVhdCookie(byte[] cookie) + { + var vhdCookie = new VhdCookie(VhdCookieType.Footer, cookie); + if (!vhdCookie.IsValid()) + throw new VhdParsingException(String.Format("Invalid Vhd footer cookie:{0}",vhdCookie.StringData)); + return vhdCookie; + } + + private IAsyncResult BeginReadVhdCookie(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadBytes(this.GetFooterOffset() + attribute.Offset, attribute.Size, callback, state); + } + + private VhdCookie EndReadVhdCookie(IAsyncResult result) + { + byte[] cookie = dataReader.EndReadBytes(result); + return CreateVhdCookie(cookie); + } + + long GetFooterOffset() + { + return dataReader.Size - VhdConstants.VHD_FOOTER_SIZE; + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/Persistence/VhdFooterSerializer.cs b/src/CLU/VhdManagement/Model/Persistence/VhdFooterSerializer.cs new file mode 100644 index 000000000000..21fbe5cf4e56 --- /dev/null +++ b/src/CLU/VhdManagement/Model/Persistence/VhdFooterSerializer.cs @@ -0,0 +1,87 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System.IO; +using System.Text; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model.Persistence +{ + public class VhdFooterSerializer + { + private readonly VhdFooter vhdFooter; + private AttributeHelper attributeHelper; + + public VhdFooterSerializer(VhdFooter vhdFooter) + { + this.vhdFooter = vhdFooter; + this.attributeHelper = new AttributeHelper(); + } + + public byte[] ToByteArray() + { + var buffer = new byte[attributeHelper.GetEntityAttribute().Size]; + using (var stream = new MemoryStream(buffer)) + { + var writer = new BinaryWriter(stream); + var dataWriter = new VhdDataWriter(writer); + dataWriter.WriteBytes(attributeHelper.GetAttribute(() => vhdFooter.Cookie).Offset, vhdFooter.Cookie.Data); + + dataWriter.WriteUInt(attributeHelper.GetAttribute(() => vhdFooter.Features).Offset, (uint)vhdFooter.Features); + dataWriter.WriteInt(attributeHelper.GetAttribute(() => vhdFooter.FileFormatVersion).Offset, + (int)vhdFooter.FileFormatVersion.Data); + dataWriter.WriteLong(attributeHelper.GetAttribute(() => vhdFooter.HeaderOffset).Offset, vhdFooter.HeaderOffset); + dataWriter.WriteTimeStamp(attributeHelper.GetAttribute(() => vhdFooter.TimeStamp).Offset, vhdFooter.TimeStamp); + dataWriter.WriteBytes(attributeHelper.GetAttribute(() => vhdFooter.CreatorApplication).Offset, + Encoding.ASCII.GetBytes(vhdFooter.CreatorApplication)); + dataWriter.WriteUInt(attributeHelper.GetAttribute(() => vhdFooter.CreatorVersion).Offset, + vhdFooter.CreatorVersion.Data); + dataWriter.WriteUInt(attributeHelper.GetAttribute(() => vhdFooter.CreatorHostOsType).Offset, + (uint)vhdFooter.CreatorHostOsType); + dataWriter.WriteLong(attributeHelper.GetAttribute(() => vhdFooter.PhsyicalSize).Offset, vhdFooter.PhsyicalSize); + dataWriter.WriteLong(attributeHelper.GetAttribute(() => vhdFooter.VirtualSize).Offset, vhdFooter.VirtualSize); + + dataWriter.SetPosition(attributeHelper.GetAttribute(() => vhdFooter.DiskGeometry).Offset); + WriteDiskGeometry(dataWriter, vhdFooter.DiskGeometry); + + dataWriter.WriteInt(attributeHelper.GetAttribute(() => vhdFooter.DiskType).Offset, (int)vhdFooter.DiskType); + dataWriter.WriteGuid(attributeHelper.GetAttribute(() => vhdFooter.UniqueId).Offset, vhdFooter.UniqueId); + dataWriter.WriteBoolean(attributeHelper.GetAttribute(() => vhdFooter.SavedState).Offset, vhdFooter.SavedState); + dataWriter.WriteBytes(attributeHelper.GetAttribute(() => vhdFooter.Reserved).Offset, vhdFooter.Reserved); + + dataWriter.WriteUInt(attributeHelper.GetAttribute(() => vhdFooter.CheckSum).Offset, ComputeCheckSum(buffer)); + } + return buffer; + } + + public uint ComputeCheckSum(byte[] buffer) + { + uint checksum = 0; + for (var i = 0; i < attributeHelper.GetEntityAttribute().Size; i++) + { + if (i < VhdConstants.VHD_FOOTER_OFFSET_CHECKSUM || i >= (VhdConstants.VHD_FOOTER_OFFSET_CHECKSUM + 4)) + { + checksum += buffer[i]; + } + } + return ~checksum; + } + + private static void WriteDiskGeometry(VhdDataWriter dataWriter, DiskGeometry diskGeometry) + { + dataWriter.WriteInt16(diskGeometry.Cylinder); + dataWriter.WriteByte(diskGeometry.Heads); + dataWriter.WriteByte(diskGeometry.Sectors); + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/Persistence/VhdHeaderFactory.cs b/src/CLU/VhdManagement/Model/Persistence/VhdHeaderFactory.cs new file mode 100644 index 000000000000..fa441c053845 --- /dev/null +++ b/src/CLU/VhdManagement/Model/Persistence/VhdHeaderFactory.cs @@ -0,0 +1,413 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using Microsoft.WindowsAzure.Commands.Tools.Common.General; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model.Persistence +{ + public class VhdHeaderFactory + { + private readonly VhdDataReader dataReader; + private readonly VhdFooter footer; + private long headerOffset; + + public VhdHeaderFactory(VhdDataReader dataReader, VhdFooter footer) + { + this.dataReader = dataReader; + this.footer = footer; + headerOffset = this.footer.HeaderOffset; + } + + public VhdHeader CreateHeader() + { + if (footer.DiskType != DiskType.Dynamic && footer.DiskType != DiskType.Differencing) + return null; + + try + { + var attributeHelper = new AttributeHelper(); + var header = new VhdHeader(); + header.Cookie = ReadHeaderCookie(attributeHelper.GetAttribute(() => header.Cookie)); + header.DataOffset = ReadDataOffset(attributeHelper.GetAttribute(() => header.DataOffset)); + header.TableOffset = ReadBATOffset(attributeHelper.GetAttribute(() => header.TableOffset)); + header.HeaderVersion = ReaderHeaderVersion(attributeHelper.GetAttribute(() => header.HeaderVersion)); + header.MaxTableEntries = ReadMaxTableEntries(attributeHelper.GetAttribute(() => header.MaxTableEntries)); + header.BlockSize = ReadBlockSize(attributeHelper.GetAttribute(() => header.BlockSize)); + header.CheckSum = ReadCheckSum(attributeHelper.GetAttribute(() => header.CheckSum)); + header.ParentUniqueId = ReadParentUniqueId(attributeHelper.GetAttribute(() => header.ParentUniqueId)); + header.ParentTimeStamp = ReadParentTimeStamp(attributeHelper.GetAttribute(() => header.ParentTimeStamp)); + header.Reserverd1 = ReadReserved1(attributeHelper.GetAttribute(() => header.Reserverd1)); + header.ParentPath = ReadParentPath(attributeHelper.GetAttribute(() => header.ParentPath)); + header.ParentLocators = ReadParentLocators(attributeHelper.GetAttribute(() => header.ParentLocators)); + header.RawData = ReadRawData(attributeHelper.GetAttribute(() => header.RawData)); + return header; + } + catch (EndOfStreamException e) + { + throw new VhdParsingException("unsupported format", e); + } + } + + private T TryCatch(Func method, IAsyncResult result) + { + try + { + return method(result); + } + catch (EndOfStreamException e) + { + throw new VhdParsingException("unsupported format", e); + } + } + + public IAsyncResult BeginCreateHeader(AsyncCallback callback, object state) + { + return AsyncMachine.BeginAsyncMachine(CreateVhdHeader, callback, state); + } + + public VhdHeader EndCreateHeader(IAsyncResult result) + { + return AsyncMachine.EndAsyncMachine(result); + } + + private IEnumerable CreateVhdHeader(AsyncMachine machine) + { + if (footer.DiskType != DiskType.Dynamic && footer.DiskType != DiskType.Differencing) + { + machine.ParameterValue = null; + yield break; + } + + var attributeHelper = new AttributeHelper(); + var header = new VhdHeader(); + + BeginReadHeaderCookie(attributeHelper.GetAttribute(() => header.Cookie), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + header.Cookie = TryCatch(EndReadHeaderCookie, machine.CompletionResult); + + BeginReadDataOffset(attributeHelper.GetAttribute(() => header.DataOffset), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + header.DataOffset = TryCatch(EndReadDataOffset, machine.CompletionResult); + + BeginReadBATOffset(attributeHelper.GetAttribute(() => header.TableOffset), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + header.TableOffset = TryCatch(EndReadBATOffset, machine.CompletionResult); + + BeginReadHeaderVersion(attributeHelper.GetAttribute(() => header.HeaderVersion), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + header.HeaderVersion = TryCatch(EndReadHeaderVersion, machine.CompletionResult); + + BeginReadMaxTableEntries(attributeHelper.GetAttribute(() => header.MaxTableEntries), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + header.MaxTableEntries = TryCatch(EndReadMaxTableEntries, machine.CompletionResult); + + BeginReadBlockSize(attributeHelper.GetAttribute(() => header.BlockSize), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + header.BlockSize = TryCatch(EndReadBlockSize, machine.CompletionResult); + + BeginReadCheckSum(attributeHelper.GetAttribute(() => header.CheckSum), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + header.CheckSum = TryCatch(EndReadCheckSum, machine.CompletionResult); + + BeginReadParentUniqueId(attributeHelper.GetAttribute(() => header.ParentUniqueId), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + header.ParentUniqueId = TryCatch(EndReadParentUniqueId, machine.CompletionResult); + + BeginReadParentTimeStamp(attributeHelper.GetAttribute(() => header.ParentTimeStamp), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + header.ParentTimeStamp = TryCatch(EndReadParentTimeStamp, machine.CompletionResult); + + BeginReadReserved1(attributeHelper.GetAttribute(() => header.Reserverd1), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + header.Reserverd1 = TryCatch(EndReadReserved1, machine.CompletionResult); + + BeginReadParentPath(attributeHelper.GetAttribute(() => header.ParentPath), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + header.ParentPath = TryCatch(EndReadParentPath, machine.CompletionResult); + + BeginReadParentLocators(attributeHelper.GetAttribute(() => header.ParentLocators), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + header.ParentLocators = TryCatch>(EndReadParentLocators, machine.CompletionResult); + + BeginReadRawData(attributeHelper.GetAttribute(() => header.RawData), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + header.RawData = TryCatch(EndReadRawData, machine.CompletionResult); + + machine.ParameterValue = header; + } + + private byte[] ReadRawData(VhdPropertyAttribute attribute) + { + return dataReader.ReadBytes(headerOffset + attribute.Offset, attribute.Size); + } + + private IAsyncResult BeginReadRawData(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadBytes(headerOffset + attribute.Offset, attribute.Size, callback, state); + } + + private byte[] EndReadRawData(IAsyncResult result) + { + var value = dataReader.EndReadBytes(result); + return (byte[]) value; + } + + private IList ReadParentLocators(VhdPropertyAttribute attribute) + { + var parentLocators = new List(); + var attributeHelper = new AttributeHelper(); + var entityAttribute = attributeHelper.GetEntityAttribute(); + + long baseOffset = headerOffset + attribute.Offset; + + for (int i = 0; i < attribute.Count; i++) + { + var parentLocatorFactory = new VhdParentLocatorFactory(dataReader, baseOffset); + parentLocators.Add(parentLocatorFactory.Create()); + baseOffset += entityAttribute.Size; + } + return parentLocators; + } + + private IAsyncResult BeginReadParentLocators(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return AsyncMachine>.BeginAsyncMachine(CreateParentLocators, attribute, callback, state); + } + + private IEnumerable CreateParentLocators(AsyncMachine> machine, VhdPropertyAttribute attribute) + { + var parentLocators = new List(); + var attributeHelper = new AttributeHelper(); + var entityAttribute = attributeHelper.GetEntityAttribute(); + + long baseOffset = headerOffset + attribute.Offset; + + for (int i = 0; i < attribute.Count; i++) + { + var parentLocatorFactory = new VhdParentLocatorFactory(dataReader, baseOffset); + + parentLocatorFactory.BeginReadCreate(machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + ParentLocator parentLocator = parentLocatorFactory.EndReadCreate(machine.CompletionResult); + + parentLocators.Add(parentLocator); + baseOffset += entityAttribute.Size; + } + machine.ParameterValue = parentLocators; + } + + private IList EndReadParentLocators(IAsyncResult result) + { + return AsyncMachine>.EndAsyncMachine(result); + } + + private uint ReadReserved1(VhdPropertyAttribute attribute) + { + return dataReader.ReadUInt32(headerOffset + attribute.Offset); + } + + private IAsyncResult BeginReadReserved1(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadUInt32(headerOffset + attribute.Offset, callback, state); + } + + private uint EndReadReserved1(IAsyncResult result) + { + var value = dataReader.EndReadUInt32(result); + return (uint) value; + } + + private uint ReadCheckSum(VhdPropertyAttribute attribute) + { + return dataReader.ReadUInt32(headerOffset + attribute.Offset); + } + + private IAsyncResult BeginReadCheckSum(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadUInt32(headerOffset + attribute.Offset, callback, state); + } + + private uint EndReadCheckSum(IAsyncResult result) + { + var value = dataReader.EndReadUInt32(result); + return (uint) value; + } + + private long ReadDataOffset(VhdPropertyAttribute attribute) + { + return (long) dataReader.ReadUInt64(headerOffset + attribute.Offset); + } + + private IAsyncResult BeginReadDataOffset(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadUInt64(headerOffset + attribute.Offset, callback, state); + } + + private long EndReadDataOffset(IAsyncResult result) + { + var value = dataReader.EndReadUInt64(result); + return (long) value; + } + + + private string ReadParentPath(VhdPropertyAttribute attribute) + { + var parentNameBytes = dataReader.ReadBytes(headerOffset + attribute.Offset, attribute.Size); + return Encoding.BigEndianUnicode.GetString(parentNameBytes).TrimEnd('\0'); + } + + private IAsyncResult BeginReadParentPath(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadBytes(headerOffset + attribute.Offset, attribute.Size, callback, state); + } + + private string EndReadParentPath(IAsyncResult result) + { + var value = dataReader.EndReadBytes(result); + return Encoding.BigEndianUnicode.GetString(value).TrimEnd('\0'); + } + + private DateTime ReadParentTimeStamp(VhdPropertyAttribute attribute) + { + return dataReader.ReadDateTime(headerOffset + attribute.Offset); + } + + private IAsyncResult BeginReadParentTimeStamp(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadDateTime(headerOffset + attribute.Offset, callback, state); + } + + private DateTime EndReadParentTimeStamp(IAsyncResult result) + { + var value = dataReader.EndReadDateTime(result); + return (DateTime) value; + } + + private Guid ReadParentUniqueId(VhdPropertyAttribute attribute) + { + return dataReader.ReadGuid(headerOffset + attribute.Offset); + } + + private IAsyncResult BeginReadParentUniqueId(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadGuid(headerOffset + attribute.Offset, callback, state); + } + + private Guid EndReadParentUniqueId(IAsyncResult result) + { + var value = dataReader.EndReadGuid(result); + return (Guid) value; + } + + private uint ReadBlockSize(VhdPropertyAttribute attribute) + { + return dataReader.ReadUInt32(headerOffset + attribute.Offset); + } + + private IAsyncResult BeginReadBlockSize(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadUInt32(headerOffset + attribute.Offset, callback, state); + } + + private uint EndReadBlockSize(IAsyncResult result) + { + var value = dataReader.EndReadUInt32(result); + return (uint) value; + } + + private uint ReadMaxTableEntries(VhdPropertyAttribute attribute) + { + return dataReader.ReadUInt32(headerOffset + attribute.Offset); + } + + private IAsyncResult BeginReadMaxTableEntries(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadUInt32(headerOffset + attribute.Offset, callback, state); + } + + private uint EndReadMaxTableEntries(IAsyncResult result) + { + var value = dataReader.EndReadUInt32(result); + return (uint) value; + } + + private long ReadBATOffset(VhdPropertyAttribute attribute) + { + return (long) dataReader.ReadUInt64(headerOffset + attribute.Offset); + } + + private IAsyncResult BeginReadBATOffset(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadUInt64(headerOffset + attribute.Offset, callback, state); + } + + private long EndReadBATOffset(IAsyncResult result) + { + var value = dataReader.EndReadUInt64(result); + return (long) value; + } + + private VhdHeaderVersion ReaderHeaderVersion(VhdPropertyAttribute attribute) + { + var version = dataReader.ReadUInt32(headerOffset + attribute.Offset); + var headerVersion = new VhdHeaderVersion(version); + if (!headerVersion.IsSupported()) + throw new VhdParsingException("unsupported format"); + return headerVersion; + } + + private IAsyncResult BeginReadHeaderVersion(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadUInt32(headerOffset + attribute.Offset, callback, state); + } + + private VhdHeaderVersion EndReadHeaderVersion(IAsyncResult result) + { + var value = dataReader.EndReadUInt32(result); + var headerVersion = new VhdHeaderVersion(value); + if (!headerVersion.IsSupported()) + throw new VhdParsingException("unsupported format"); + return headerVersion; + } + + private VhdCookie ReadHeaderCookie(VhdPropertyAttribute attribute) + { + var cookie = dataReader.ReadBytes(headerOffset + attribute.Offset, attribute.Size); + var vhdCookie = new VhdCookie(VhdCookieType.Header, cookie); + if (!vhdCookie.IsValid()) + throw new VhdParsingException(String.Format("unsupported format, Cookie:{0}", vhdCookie)); + return vhdCookie; + } + + private IAsyncResult BeginReadHeaderCookie(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadBytes(headerOffset + attribute.Offset, attribute.Size, callback, state); + } + + private VhdCookie EndReadHeaderCookie(IAsyncResult result) + { + var value = dataReader.EndReadBytes(result); + var vhdCookie = new VhdCookie(VhdCookieType.Header, value); + if (!vhdCookie.IsValid()) + throw new VhdParsingException(String.Format("unsupported format, Cookie:{0}", vhdCookie)); + return vhdCookie; + } + + + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/Persistence/VhdParentLocatorFactory.cs b/src/CLU/VhdManagement/Model/Persistence/VhdParentLocatorFactory.cs new file mode 100644 index 000000000000..15ef804be9f7 --- /dev/null +++ b/src/CLU/VhdManagement/Model/Persistence/VhdParentLocatorFactory.cs @@ -0,0 +1,210 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Text; +using Microsoft.WindowsAzure.Commands.Tools.Common.General; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model.Persistence +{ + public class VhdParentLocatorFactory + { + private readonly VhdDataReader dataReader; + private readonly long offset; + private AttributeHelper attributeHelper; + + public VhdParentLocatorFactory(VhdDataReader dataReader, long offset) + { + this.dataReader = dataReader; + this.offset = offset; + attributeHelper = new AttributeHelper(); + } + + public ParentLocator Create() + { + var locator = new ParentLocator(); + locator.PlatformCode = ReadPlaformCode(attributeHelper.GetAttribute(() => locator.PlatformCode)); + locator.PlatformDataSpace = ReadPlatformDataSpace(attributeHelper.GetAttribute(() => locator.PlatformDataSpace)); + locator.PlatformDataLength = ReadPlatformDataLength(attributeHelper.GetAttribute(() => locator.PlatformDataLength)); + locator.Reserved = ReadReserved(attributeHelper.GetAttribute(() => locator.Reserved)); + locator.PlatformDataOffset = ReadPlatformDataOffset(attributeHelper.GetAttribute(() => locator.PlatformDataOffset)); + locator.PlatformSpecificFileLocator = ReadFileLocator(locator); + return locator; + } + + public IAsyncResult BeginReadCreate(AsyncCallback callback, object state) + { + return AsyncMachine.BeginAsyncMachine(CreateParentLocator, callback, state); + } + + public ParentLocator EndReadCreate(IAsyncResult result) + { + return AsyncMachine.EndAsyncMachine(result); + } + + private IEnumerable CreateParentLocator(AsyncMachine machine) + { + var locator = new ParentLocator(); + + BeginReadPlatformCode(attributeHelper.GetAttribute(() => locator.PlatformCode), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + locator.PlatformCode = EndReadPlatformCode(machine.CompletionResult); + + BeginReadPlatformDataSpace(attributeHelper.GetAttribute(() => locator.PlatformDataSpace), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + locator.PlatformDataSpace = EndReadPlatformDataSpace(machine.CompletionResult); + + BeginReadPlatformDataLength(attributeHelper.GetAttribute(() => locator.PlatformDataLength), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + locator.PlatformDataLength = EndReadPlatformDataLength(machine.CompletionResult); + + BeginReadReserved(attributeHelper.GetAttribute(() => locator.Reserved), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + locator.Reserved = EndReadReserved(machine.CompletionResult); + + BeginReadPlatformDataOffset(attributeHelper.GetAttribute(() => locator.PlatformDataOffset), machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + locator.PlatformDataOffset = EndReadPlatformDataOffset(machine.CompletionResult); + + BeginReadFileLocator(locator, machine.CompletionCallback, null); + yield return CompletionPort.SingleOperation; + locator.PlatformSpecificFileLocator = EndReadFileLocator(machine.CompletionResult); + + machine.ParameterValue = locator; + } + + private string ReadFileLocator(ParentLocator locator) + { + var fileLocator = dataReader.ReadBytes(locator.PlatformDataOffset, locator.PlatformDataLength); + return CreateFileLocator(locator, fileLocator); + } + + private string CreateFileLocator(ParentLocator locator, byte[] fileLocator) + { + switch(locator.PlatformCode) + { + case PlatformCode.None: + return String.Empty; + case PlatformCode.Wi2R: + case PlatformCode.Wi2K: + throw new VhdParsingException(String.Format("Deprecated PlatformCode:{0}",locator.PlatformCode)); + case PlatformCode.W2Ru: + //TODO: Add differencing disks path name, this is relative path + return Encoding.Unicode.GetString(fileLocator); + case PlatformCode.W2Ku: + return Encoding.Unicode.GetString(fileLocator); + case PlatformCode.Mac: + //TODO: Mac OS alias stored as a blob? + throw new NotImplementedException(String.Format("PlatformCode: {0}", locator.PlatformCode)); + case PlatformCode.MacX: + return Encoding.UTF8.GetString(fileLocator); + } + return Encoding.BigEndianUnicode.GetString(fileLocator).TrimEnd('\0'); + } + + private IAsyncResult BeginReadFileLocator(ParentLocator locator, AsyncCallback callback, object state) + { + return dataReader.BeginReadBytes(locator.PlatformDataOffset, locator.PlatformDataLength, callback, locator); + } + + private string EndReadFileLocator(IAsyncResult result) + { + var fileLocator = dataReader.EndReadBytes(result); + var locator = (ParentLocator) result.AsyncState; + return CreateFileLocator(locator, fileLocator); + } + + private PlatformCode ReadPlaformCode(VhdPropertyAttribute attribute) + { + return (PlatformCode) dataReader.ReadUInt32(offset + attribute.Offset); + } + + private IAsyncResult BeginReadPlatformCode(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadUInt32(offset + attribute.Offset, callback, state); + } + + private PlatformCode EndReadPlatformCode(IAsyncResult result) + { + var value = dataReader.EndReadUInt32(result); + return (PlatformCode) value; + } + + private int ReadPlatformDataSpace(VhdPropertyAttribute attribute) + { + return (int)dataReader.ReadUInt32(offset + attribute.Offset); + } + + private IAsyncResult BeginReadPlatformDataSpace(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadUInt32(offset + attribute.Offset, callback, state); + } + + private int EndReadPlatformDataSpace(IAsyncResult result) + { + var value = dataReader.EndReadUInt32(result); + return (int) value; + } + + private int ReadPlatformDataLength(VhdPropertyAttribute attribute) + { + return (int)dataReader.ReadUInt32(offset + attribute.Offset); + } + + private IAsyncResult BeginReadPlatformDataLength(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadUInt32(offset + attribute.Offset, callback, state); + } + + private int EndReadPlatformDataLength(IAsyncResult result) + { + var value = dataReader.EndReadUInt32(result); + return (int) value; + } + + private int ReadReserved(VhdPropertyAttribute attribute) + { + return (int)dataReader.ReadUInt32(offset + attribute.Offset); + } + + private IAsyncResult BeginReadReserved(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadUInt32(offset + attribute.Offset, callback, state); + } + + private int EndReadReserved(IAsyncResult result) + { + var value = dataReader.EndReadUInt32(result); + return (int) value; + } + + private long ReadPlatformDataOffset(VhdPropertyAttribute attribute) + { + return (long) dataReader.ReadUInt64(offset + attribute.Offset); + } + + private IAsyncResult BeginReadPlatformDataOffset(VhdPropertyAttribute attribute, AsyncCallback callback, object state) + { + return dataReader.BeginReadUInt64(offset + attribute.Offset, callback, state); + } + + private long EndReadPlatformDataOffset(IAsyncResult result) + { + var value = dataReader.EndReadUInt64(result); + return (long) value; + } + + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/PlatformCode.cs b/src/CLU/VhdManagement/Model/PlatformCode.cs new file mode 100644 index 000000000000..b28d3191d2d2 --- /dev/null +++ b/src/CLU/VhdManagement/Model/PlatformCode.cs @@ -0,0 +1,27 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model +{ + public enum PlatformCode + { + None = 0x0, + Wi2R = 0x57693272, + Wi2K = 0x5769326B, + W2Ru = 0x57327275, + W2Ku = 0x57326B75, + Mac = 0x4D616320, + MacX = 0x4D616358 + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/Sector.cs b/src/CLU/VhdManagement/Model/Sector.cs new file mode 100644 index 000000000000..6ef4f41d8f90 --- /dev/null +++ b/src/CLU/VhdManagement/Model/Sector.cs @@ -0,0 +1,25 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model +{ + public class Sector + { + public uint BlockIndex { get; set; } + public long SectorIndex { get; set; } + public byte[] Data { get; set; } + public IndexRange LogicalRange { get; set; } + public long GlobalSectorIndex { get; set; } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/VhdCookie.cs b/src/CLU/VhdManagement/Model/VhdCookie.cs new file mode 100644 index 000000000000..7deeb2025dff --- /dev/null +++ b/src/CLU/VhdManagement/Model/VhdCookie.cs @@ -0,0 +1,79 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Linq; +using System.Text; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model +{ + public class VhdCookie + { + static readonly byte[] FooterCookie = Encoding.ASCII.GetBytes("conectix"); + static readonly byte[] HeaderCookie = Encoding.ASCII.GetBytes("cxsparse"); + + private readonly VhdCookieType cookieType; + private readonly byte[] expectedData; + + public static VhdCookie CreateFooterCookie() + { + return new VhdCookie(VhdCookieType.Footer, FooterCookie); + } + + public static VhdCookie CreateHeaderCookie() + { + return new VhdCookie(VhdCookieType.Header, HeaderCookie); + } + + public VhdCookie(VhdCookieType cookieType, byte[] data) + { + this.cookieType = cookieType; + this.Data = data; + this.expectedData = GetExpectedCookie(); + } + + public byte[] Data { get; private set; } + + public string StringData + { + get { return Encoding.ASCII.GetString(this.Data); } + } + + public bool IsValid() + { + if (Data.Length != expectedData.Length) + { + return false; + } + return !expectedData.Where((t, i) => Data[i] != t).Any(); + } + + private byte[] GetExpectedCookie() + { + return cookieType == VhdCookieType.Header ? HeaderCookie : FooterCookie; + } + + public VhdCookie CreateCopy() + { + var copy = new byte[Data.Length]; + Array.Copy(Data, copy, Data.Length); + return new VhdCookie(this.cookieType, copy); + } + + public override string ToString() + { + return StringData; + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/VhdCookieType.cs b/src/CLU/VhdManagement/Model/VhdCookieType.cs new file mode 100644 index 000000000000..ca54a12b924d --- /dev/null +++ b/src/CLU/VhdManagement/Model/VhdCookieType.cs @@ -0,0 +1,22 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model +{ + public enum VhdCookieType + { + Header, + Footer + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/VhdCreatorVersion.cs b/src/CLU/VhdManagement/Model/VhdCreatorVersion.cs new file mode 100644 index 000000000000..2327344fdee7 --- /dev/null +++ b/src/CLU/VhdManagement/Model/VhdCreatorVersion.cs @@ -0,0 +1,35 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model +{ + public class VhdCreatorVersion + { + public static VhdCreatorVersion VS2004 = new VhdCreatorVersion(0x00010000); + public static VhdCreatorVersion VPC2004 = new VhdCreatorVersion(0x00050000); + public static VhdCreatorVersion CSUP2011 = new VhdCreatorVersion(0x00070000); + + public VhdCreatorVersion(uint data) + { + this.Data = data; + } + + public uint Data { get; private set; } + + public override string ToString() + { + return this.Data.ToString(); + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/VhdEntityAttribute.cs b/src/CLU/VhdManagement/Model/VhdEntityAttribute.cs new file mode 100644 index 000000000000..ab428746b70c --- /dev/null +++ b/src/CLU/VhdManagement/Model/VhdEntityAttribute.cs @@ -0,0 +1,23 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model +{ + public class VhdEntityAttribute : Attribute + { + public int Size { get; set; } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/VhdEntityDescriptor.cs b/src/CLU/VhdManagement/Model/VhdEntityDescriptor.cs new file mode 100644 index 000000000000..debfb64f50fb --- /dev/null +++ b/src/CLU/VhdManagement/Model/VhdEntityDescriptor.cs @@ -0,0 +1,50 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.Linq; +using System.Reflection; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model +{ + public class VhdEntityDescriptor + { + public VhdEntityDescriptor() + { + this.PropertyDescriptors = GetPropertyDescriptors(); + } + + public IList PropertyDescriptors { get; private set; } + + static IList GetPropertyDescriptors() + { + return (from p in typeof(T).GetProperties(BindingFlags.Instance | BindingFlags.Public) + let vhdPropertyAttributes = p.GetCustomAttributes(typeof(VhdPropertyAttribute), false) + // TODO: CLU + let exists = vhdPropertyAttributes.Count() > 0 + //let exists = vhdPropertyAttributes.Length > 0 + let getter = p.GetGetMethod(true) + let setter = p.GetSetMethod(true) + where exists + select new VhdPropertyDescriptor + { + // TODO: CLU + Attribute = (VhdPropertyAttribute)(vhdPropertyAttributes.ElementAt(0)), + //Attribute = (VhdPropertyAttribute)(vhdPropertyAttributes[0]), + Getter = getter, + Setter = setter + }).ToList(); + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/VhdFeature.cs b/src/CLU/VhdManagement/Model/VhdFeature.cs new file mode 100644 index 000000000000..be79f7dce86b --- /dev/null +++ b/src/CLU/VhdManagement/Model/VhdFeature.cs @@ -0,0 +1,26 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model +{ + [Flags] + public enum VhdFeature : uint + { + NoFeaturesEnabled = 0x00000000, + Temporary = 0x00000001, + Reserved = 0x00000002 + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/VhdFile.cs b/src/CLU/VhdManagement/Model/VhdFile.cs new file mode 100644 index 000000000000..916192f5f93d --- /dev/null +++ b/src/CLU/VhdManagement/Model/VhdFile.cs @@ -0,0 +1,108 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using Microsoft.WindowsAzure.Commands.Tools.Vhd.Model.Persistence; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model +{ + public class VhdFile : IDisposable + { + private BinaryReader reader; + private bool disposed; + + public VhdFile(VhdFooter footer, VhdHeader header, BlockAllocationTable bat, VhdFile parent, Stream stream) + { + this.Footer = footer; + this.Header = header; + this.BlockAllocationTable = bat; + this.Parent = parent; + this.reader = new BinaryReader(stream, Encoding.Unicode); + DataReader = new VhdDataReader(this.reader); + } + + // These properties, and all others on this object, must remain immutable for certain FxCop suppressions to be allowed. + // If at any time this class is changed, please revisit all suppressions of the DoNotDeclareReadOnlyMutableReferenceTypes + // rule where VhdFile is declared as a readonly object. Otherwise it is always safe to suppress the rule DoNotDeclareReadOnlyMutableReferenceTypes + public VhdDataReader DataReader { get; private set; } + public VhdFooter Footer { get; private set; } + public VhdHeader Header { get; private set; } + public BlockAllocationTable BlockAllocationTable { get; private set; } + public DiskType DiskType { get { return Footer.DiskType; } } + public VhdFile Parent { get; private set; } + + public IEnumerable GetBlocks() + { + var blockFactory = this.GetBlockFactory(); + for (long index = 0; index < blockFactory.BlockCount; index++) + { + yield return blockFactory.Create((uint)index); + } + } + + public IBlockFactory GetBlockFactory() + { + switch (this.DiskType) + { + case DiskType.Fixed: + return new FixedDiskBlockFactory(this); + case DiskType.Dynamic: + return new DynamicDiskBlockFactory(this); + case DiskType.Differencing: + return new DifferencingDiskBlockFactory(this); + default: + throw new InvalidOperationException(String.Format("Unsupported DiskType:{0}", this.DiskType)); + } + } + + public IEnumerable GetIdentityChain() + { + var identities = new List { this.Footer.UniqueId }; + var currentFile = this.Parent; + while (currentFile != null) + { + identities.Add(currentFile.Footer.UniqueId); + currentFile = currentFile.Parent; + } + return identities; + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (!disposed) + { + if (disposing) + { + // TODO: CLU + this.reader.Dispose(); + //this.reader.Close(); + } + if(Parent != null) + { + Parent.Dispose(); + } + disposed = true; + } + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/VhdFileFormatVersion.cs b/src/CLU/VhdManagement/Model/VhdFileFormatVersion.cs new file mode 100644 index 000000000000..1850a98e2a69 --- /dev/null +++ b/src/CLU/VhdManagement/Model/VhdFileFormatVersion.cs @@ -0,0 +1,55 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model +{ + + public class VhdFileFormatVersion + { + public static VhdFileFormatVersion DefaultFileFormatVersion = + new VhdFileFormatVersion((int)VHD_FOOTER_SUPPORTED_VERSION); + + const uint VHD_FOOTER_SUPPORTED_VERSION = 0x00010000; + + public VhdFileFormatVersion(uint data) + { + this.Data = data; + } + + public uint Data { get; private set; } + + public bool IsSupported() + { + return Data == VHD_FOOTER_SUPPORTED_VERSION; + } + + public override string ToString() + { + return this.Data.ToString(); + } + + public override bool Equals(object obj) + { + if (ReferenceEquals(null, obj)) return false; + if (ReferenceEquals(this, obj)) return true; + if (obj.GetType() != typeof(VhdFileFormatVersion)) return false; + return ((VhdFileFormatVersion)obj).Data == Data; + } + + public override int GetHashCode() + { + return Data.GetHashCode(); + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/VhdFooter.cs b/src/CLU/VhdManagement/Model/VhdFooter.cs new file mode 100644 index 000000000000..07effe50c7e2 --- /dev/null +++ b/src/CLU/VhdManagement/Model/VhdFooter.cs @@ -0,0 +1,141 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model +{ + [VhdEntity(Size = 512)] + public class VhdFooter + { + [VhdProperty(Offset = 0, Size = 8)] + public VhdCookie Cookie { get; set; } + + [VhdProperty(Offset = 8, Size = 4)] + public VhdFeature Features { get; set; } + + [VhdProperty(Offset = 12, Size = 4)] + public VhdFileFormatVersion FileFormatVersion { get; set; } + + [VhdProperty(Offset = 16, Size = 8)] + public long HeaderOffset { get; set; } + + [VhdProperty(Offset = 24, Size = 4)] + public DateTime TimeStamp { get; set; } + + [VhdProperty(Offset = 28, Size = 4)] + public string CreatorApplication { get; set; } + + [VhdProperty(Offset = 32, Size = 4)] + public VhdCreatorVersion CreatorVersion { get; set; } + + [VhdProperty(Offset = 36, Size = 4)] + public HostOsType CreatorHostOsType { get; set; } + + [VhdProperty(Offset = 40, Size = 8)] + public long PhsyicalSize { get; set; } + + [VhdProperty(Offset = 48, Size = 8)] + public long VirtualSize { get; set; } + + [VhdProperty(Offset = 56, Size = 4)] + public DiskGeometry DiskGeometry { get; set; } + + [VhdProperty(Offset = 60, Size = 4)] + public DiskType DiskType { get; set; } + + [VhdProperty(Offset = 64, Size = 4)] + public uint CheckSum { get; set; } + + [VhdProperty(Offset = 68, Size = 16)] + public Guid UniqueId { get; set; } + + [VhdProperty(Offset = 84, Size = 1)] + public bool SavedState { get; set; } + + [VhdProperty(Offset = 85, Size = 427)] + public byte[] Reserved { get; set; } + + [VhdProperty(Offset = 0, Size = 512)] + public byte[] RawData { get; set; } + + public VhdFooter CreateCopy() + { + return new VhdFooter + { + Cookie = this.Cookie.CreateCopy(), + Features = this.Features, + FileFormatVersion = this.FileFormatVersion, + HeaderOffset = this.HeaderOffset, + TimeStamp = this.TimeStamp, + CreatorApplication = this.CreatorApplication, + CreatorVersion = this.CreatorVersion, + CreatorHostOsType = this.CreatorHostOsType, + PhsyicalSize = this.PhsyicalSize, + VirtualSize = this.VirtualSize, + DiskGeometry = this.DiskGeometry.CreateCopy(), + DiskType = this.DiskType, + CheckSum = this.CheckSum, + UniqueId = this.UniqueId, + SavedState = this.SavedState, + Reserved = CreateCopy(this.Reserved), + RawData = CreateCopy(this.RawData), + }; + } + + static byte[] CreateCopy(byte[] data) + { + var result = new byte[data.Length]; + Array.Copy(data, result, data.Length); + return result; + } + + public override bool Equals(object obj) + { + return Equals(obj as VhdFooter); + } + + private bool Equals(VhdFooter other) + { + if (ReferenceEquals(null, other)) return false; + if (ReferenceEquals(this, other)) return true; + return Equals(other.Cookie, Cookie) && Equals(other.Features, Features) && Equals(other.FileFormatVersion, FileFormatVersion) && other.HeaderOffset == HeaderOffset && other.TimeStamp.Equals(TimeStamp) && Equals(other.CreatorApplication, CreatorApplication) && Equals(other.CreatorVersion, CreatorVersion) && Equals(other.CreatorHostOsType, CreatorHostOsType) && other.PhsyicalSize == PhsyicalSize && other.VirtualSize == VirtualSize && Equals(other.DiskGeometry, DiskGeometry) && Equals(other.DiskType, DiskType) && other.CheckSum == CheckSum && other.UniqueId.Equals(UniqueId) && other.SavedState.Equals(SavedState) && Equals(other.Reserved, Reserved) && Equals(other.RawData, RawData); + } + + public override int GetHashCode() + { + unchecked + { + int result = (Cookie != null ? Cookie.GetHashCode() : 0); + result = (result*397) ^ Features.GetHashCode(); + result = (result*397) ^ (FileFormatVersion != null ? FileFormatVersion.GetHashCode() : 0); + result = (result*397) ^ HeaderOffset.GetHashCode(); + result = (result*397) ^ TimeStamp.GetHashCode(); + result = (result*397) ^ (CreatorApplication != null ? CreatorApplication.GetHashCode() : 0); + result = (result*397) ^ (CreatorVersion != null ? CreatorVersion.GetHashCode() : 0); + result = (result*397) ^ CreatorHostOsType.GetHashCode(); + result = (result*397) ^ PhsyicalSize.GetHashCode(); + result = (result*397) ^ VirtualSize.GetHashCode(); + result = (result*397) ^ (DiskGeometry != null ? DiskGeometry.GetHashCode() : 0); + result = (result*397) ^ DiskType.GetHashCode(); + result = (result*397) ^ CheckSum.GetHashCode(); + result = (result*397) ^ UniqueId.GetHashCode(); + result = (result*397) ^ SavedState.GetHashCode(); + result = (result*397) ^ (Reserved != null ? Reserved.GetHashCode() : 0); + result = (result*397) ^ (RawData != null ? RawData.GetHashCode() : 0); + return result; + } + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/VhdHeader.cs b/src/CLU/VhdManagement/Model/VhdHeader.cs new file mode 100644 index 000000000000..0feafa35d951 --- /dev/null +++ b/src/CLU/VhdManagement/Model/VhdHeader.cs @@ -0,0 +1,77 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model +{ + [VhdEntity(Size = 1024)] + public class VhdHeader + { + [VhdProperty(Offset = 0, Size = 8)] + public VhdCookie Cookie { get; set; } + + [VhdProperty(Offset = 8, Size = 8)] + public long DataOffset { get; set; } + + [VhdProperty(Offset = 16, Size = 8)] + public long TableOffset { get; set; } + + [VhdProperty(Offset = 24, Size = 4)] + public VhdHeaderVersion HeaderVersion { get; set; } + + [VhdProperty(Offset = 28, Size = 4)] + public uint MaxTableEntries { get; set; } + + [VhdProperty(Offset = 32, Size = 4)] + public uint BlockSize { get; set; } + + [VhdProperty(Offset = 36, Size = 4)] + public uint CheckSum { get; set; } + + [VhdProperty(Offset = 40, Size = 16)] + public Guid ParentUniqueId { get; set; } + + [VhdProperty(Offset = 56, Size = 4)] + public DateTime ParentTimeStamp { get; set; } + + [VhdProperty(Offset = 60, Size = 4)] + public uint Reserverd1 { get; set; } + + [VhdProperty(Offset = 64, Size = 512)] + public string ParentPath { get; set; } + + [VhdProperty(Offset = 576, Count = 8)] + public IList ParentLocators { get; set; } + + [VhdProperty(Offset = 0, Size = 1024)] + public byte[] RawData { get; set; } + + public string GetAbsoluteParentPath() + { + return (from p in ParentLocators + where p.PlatformCode == PlatformCode.W2Ku + select p.PlatformSpecificFileLocator).FirstOrDefault(); + } + + public string GetRelativeParentPath() + { + return (from p in ParentLocators + where p.PlatformCode == PlatformCode.W2Ru + select p.PlatformSpecificFileLocator).FirstOrDefault(); + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/VhdHeaderVersion.cs b/src/CLU/VhdManagement/Model/VhdHeaderVersion.cs new file mode 100644 index 000000000000..5b3731986fd9 --- /dev/null +++ b/src/CLU/VhdManagement/Model/VhdHeaderVersion.cs @@ -0,0 +1,38 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model +{ + public class VhdHeaderVersion + { + private const uint VHD_HEADER_SUPPORTED_VERSION = 0x00010000; + + public VhdHeaderVersion(uint data) + { + this.Data = data; + } + + public uint Data { get; private set; } + + public bool IsSupported() + { + return Data == VHD_HEADER_SUPPORTED_VERSION; + } + + public override string ToString() + { + return this.Data.ToString(); + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/VhdPropertyAttribute.cs b/src/CLU/VhdManagement/Model/VhdPropertyAttribute.cs new file mode 100644 index 000000000000..c57d15cc6446 --- /dev/null +++ b/src/CLU/VhdManagement/Model/VhdPropertyAttribute.cs @@ -0,0 +1,25 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model +{ + public class VhdPropertyAttribute : Attribute + { + public int Offset { get; set; } + public int Size { get; set; } + public int Count { get; set; } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/VhdPropertyDescriptor.cs b/src/CLU/VhdManagement/Model/VhdPropertyDescriptor.cs new file mode 100644 index 000000000000..13a758e8f08b --- /dev/null +++ b/src/CLU/VhdManagement/Model/VhdPropertyDescriptor.cs @@ -0,0 +1,27 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System.Reflection; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model +{ + public class VhdPropertyDescriptor + { + public VhdPropertyAttribute Attribute { get; set; } + + public MethodInfo Getter { get; set; } + + public MethodInfo Setter { get; set; } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Model/VhdTimeStamp.cs b/src/CLU/VhdManagement/Model/VhdTimeStamp.cs new file mode 100644 index 000000000000..b3f7d1702a9f --- /dev/null +++ b/src/CLU/VhdManagement/Model/VhdTimeStamp.cs @@ -0,0 +1,47 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd.Model +{ + public class VhdTimeStamp + { + private static readonly DateTime VhdBaseTime = new DateTime(2000, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); + private readonly uint value; + + public VhdTimeStamp(DateTime dateTime) + { + if (dateTime < VhdBaseTime) + { + var message = String.Format("DateTime must be after Base Vhd Time: {0}", VhdBaseTime); + throw new ArgumentOutOfRangeException("dateTime", message); + } + + this.TotalSeconds = (uint)dateTime.Subtract(VhdBaseTime).TotalSeconds; + } + + public VhdTimeStamp(uint value) + { + this.value = value; + } + + public uint TotalSeconds { get; private set; } + + public DateTime ToDateTime() + { + return VhdBaseTime.AddSeconds(value).ToUniversalTime(); + } + } +} \ No newline at end of file diff --git a/src/CLU/VhdManagement/Properties/AssemblyInfo.cs b/src/CLU/VhdManagement/Properties/AssemblyInfo.cs new file mode 100644 index 000000000000..0f4ede43c0a7 --- /dev/null +++ b/src/CLU/VhdManagement/Properties/AssemblyInfo.cs @@ -0,0 +1,37 @@ +using System; +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("VhdManagement")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("Microsoft")] +[assembly: AssemblyProduct("VhdManagement")] +[assembly: AssemblyCopyright("Copyright © Microsoft")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("f533c72c-c034-4e74-a961-8b2da198e104")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] +[assembly: CLSCompliant(false)] \ No newline at end of file diff --git a/src/CLU/VhdManagement/SparseStream.cs b/src/CLU/VhdManagement/SparseStream.cs new file mode 100644 index 000000000000..bb63a7898e0b --- /dev/null +++ b/src/CLU/VhdManagement/SparseStream.cs @@ -0,0 +1,54 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.IO; +using Microsoft.WindowsAzure.Commands.Tools.Vhd.Model; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd +{ + ///

+ /// A stream with additional extent information. + /// + /// + /// Extent information reveals the ranges of the stream that contain non-zero data. Clients may use + /// the extent information to optimize reads to the stream. The stream supports reads over any range, + /// regardless of the extents. + /// + public abstract class SparseStream : Stream + { + public abstract IEnumerable Extents { get; } + } + + /// + /// An extent. + /// + public struct StreamExtent + { + public Guid Owner; + public long StartOffset; + public long EndOffset; + + public long Length + { + get + { + return (this.EndOffset - this.StartOffset) + 1; + } + } + + public IndexRange Range { get; set; } + } +} diff --git a/src/CLU/VhdManagement/VhdManagement.xproj b/src/CLU/VhdManagement/VhdManagement.xproj new file mode 100644 index 000000000000..b48d0cb80727 --- /dev/null +++ b/src/CLU/VhdManagement/VhdManagement.xproj @@ -0,0 +1,18 @@ + + + + 14.0 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) + + + + 094a32ea-babc-4a0c-9b6c-3cf7f6eabec9 + Microsoft.WindowsAzure.Commands.Tools.Vhd + ..\artifacts\obj\$(MSBuildProjectName) + ..\artifacts\bin\$(MSBuildProjectName)\ + + + 2.0 + + + \ No newline at end of file diff --git a/src/CLU/VhdManagement/VirtualDiskStream.cs b/src/CLU/VhdManagement/VirtualDiskStream.cs new file mode 100644 index 000000000000..765f3233e1ff --- /dev/null +++ b/src/CLU/VhdManagement/VirtualDiskStream.cs @@ -0,0 +1,257 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.IO; +using Microsoft.WindowsAzure.Commands.Tools.Vhd.Model; +using Microsoft.WindowsAzure.Commands.Tools.Vhd.Model.Persistence; + +namespace Microsoft.WindowsAzure.Commands.Tools.Vhd +{ + /// + /// Provides a logical stream over a virtual hard disk (VHD). + /// + /// + /// This stream implementation provides a "view" over a VHD, such that the + /// VHD appears to be an ordinary fixed VHD file, regardless of the true physical layout. + /// This stream supports any combination of differencing, dynamic disks, and fixed disks. + /// + public class VirtualDiskStream : SparseStream + { + private long position; + private VhdFile vhdFile; + private IBlockFactory blockFactory; + private IndexRange footerRange; + private IndexRange fileDataRange; + private bool isDisposed; + + public VirtualDiskStream(string vhdPath) + { + this.vhdFile = new VhdFileFactory().Create(vhdPath); + this.blockFactory = vhdFile.GetBlockFactory(); + footerRange = this.blockFactory.GetFooterRange(); + fileDataRange = IndexRange.FromLength(0, this.Length - footerRange.Length); + } + + public override bool CanRead + { + get { return true; } + } + + public override bool CanSeek + { + get { return true; } + } + + public override bool CanWrite + { + get { return false; } + } + + public override void Flush() + { + } + + public override sealed long Length + { + get { return this.footerRange.EndIndex + 1; } + } + + public override long Position + { + get { return this.position; } + set + { + if (value < 0) throw new ArgumentException(); + if (value >= this.Length) throw new EndOfStreamException(); + this.position = value; + } + } + + /// + /// Gets the extents of the stream that contain data. + /// + public override IEnumerable Extents + { + get + { + for (uint index = 0; index < blockFactory.BlockCount; index++ ) + { + var block = blockFactory.Create(index); + if(!block.Empty) + { + yield return new StreamExtent + { + Owner = block.VhdUniqueId, + StartOffset = block.LogicalRange.StartIndex, + EndOffset = block.LogicalRange.EndIndex, + Range = block.LogicalRange + }; + } + } + yield return new StreamExtent + { + Owner = vhdFile.Footer.UniqueId, + StartOffset = this.footerRange.StartIndex, + EndOffset = this.footerRange.EndIndex, + Range = this.footerRange + }; + } + } + + public DiskType DiskType + { + get { return this.vhdFile.DiskType; } + } + + public DiskType RootDiskType + { + get + { + var diskType = this.vhdFile.DiskType; + for(var parent = this.vhdFile.Parent; parent != null; parent = parent.Parent) + { + diskType = parent.DiskType; + } + return diskType; + } + } + /// + /// Reads the specified number of bytes from the current position. + /// + public override int Read(byte[] buffer, int offset, int count) + { + if (count <= 0) + { + return 0; + } + + try + { + var rangeToRead = IndexRange.FromLength(this.position, count); + + int writtenCount = 0; + if (fileDataRange.Intersection(rangeToRead) == null) + { + int readCountFromFooter; + if (TryReadFromFooter(rangeToRead, buffer, offset, out readCountFromFooter)) + { + writtenCount += readCountFromFooter; + } + return writtenCount; + } + + rangeToRead = fileDataRange.Intersection(rangeToRead); + + var startingBlock = ByteToBlock(rangeToRead.StartIndex); + var endingBlock = ByteToBlock(rangeToRead.EndIndex); + + for(var blockIndex = startingBlock; blockIndex <= endingBlock; blockIndex++) + { + var currentBlock = blockFactory.Create(blockIndex); + var rangeToReadInBlock = currentBlock.LogicalRange.Intersection(rangeToRead); + + var copyStartIndex = rangeToReadInBlock.StartIndex % blockFactory.GetBlockSize(); + Buffer.BlockCopy(currentBlock.Data, (int) copyStartIndex, buffer, offset + writtenCount, (int) rangeToReadInBlock.Length); + + writtenCount += (int)rangeToReadInBlock.Length; + } + this.position += writtenCount; + + return writtenCount; + } + catch (Exception e) + { + throw new VhdParsingException("Invalid or Corrupted VHD file", e); + } + } + + public bool TryReadFromFooter(IndexRange rangeToRead, byte[] buffer, int offset, out int readCount) + { + readCount = 0; + var rangeToReadFromFooter = this.footerRange.Intersection(rangeToRead); + if (rangeToReadFromFooter != null) + { + var footerData = GenerateFooter(); + var copyStartIndex = rangeToReadFromFooter.StartIndex - footerRange.StartIndex; + Buffer.BlockCopy(footerData, (int)copyStartIndex, buffer, offset, (int)rangeToReadFromFooter.Length); + this.position += (int)rangeToReadFromFooter.Length; + readCount = (int)rangeToReadFromFooter.Length; + return true; + } + return false; + } + + private uint ByteToBlock(long position) + { + uint sectorsPerBlock = (uint) (this.blockFactory.GetBlockSize() / VhdConstants.VHD_SECTOR_LENGTH); + return (uint)Math.Floor((position / VhdConstants.VHD_SECTOR_LENGTH) * 1.0m / sectorsPerBlock); + } + + private byte[] GenerateFooter() + { + var footer = vhdFile.Footer.CreateCopy(); + if(vhdFile.Footer.DiskType != DiskType.Fixed) + { + footer.HeaderOffset = VhdConstants.VHD_NO_DATA_LONG; + footer.DiskType = DiskType.Fixed; + footer.CreatorApplication = VhdFooterFactory.WindowsAzureCreatorApplicationName; + } + var serializer = new VhdFooterSerializer(footer); + return serializer.ToByteArray(); + } + + public override long Seek(long offset, SeekOrigin origin) + { + switch (origin) + { + case SeekOrigin.Begin: + this.Position = offset; + break; + case SeekOrigin.Current: + this.Position += offset; + break; + case SeekOrigin.End: + this.Position -= offset; + break; + default: + throw new NotSupportedException(); + } + return this.Position; + } + + public override void SetLength(long value) + { + throw new NotSupportedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new NotSupportedException(); + } + + protected override void Dispose(bool disposing) + { + if(!isDisposed) + { + if(disposing) + { + this.vhdFile.Dispose(); + isDisposed = true; + } + } + } + } +} diff --git a/src/CLU/VhdManagement/project.json b/src/CLU/VhdManagement/project.json new file mode 100644 index 000000000000..d963c268ecef --- /dev/null +++ b/src/CLU/VhdManagement/project.json @@ -0,0 +1,56 @@ +{ + "version": "1.0.0-*", + "description": "Microsoft.WindowsAzure.Commands.Tools.Vhd", + "authors": [ "huangpf, markcowl, hovsepm, haocs" ], + "tags": [ "" ], + "projectUrl": "", + "licenseUrl": "", + "frameworks": { + "dnxcore50": { + "dependencies": { + "Microsoft.NETCore": "5.0.1-beta-23516", + "Microsoft.NETCore.Platforms": "1.0.1-beta-23516", + "Microsoft.CSharp": "4.0.1-beta-23516" + } + } + }, + "dependencies": { + "System.Linq": "4.0.1-beta-23516", + "Microsoft.CLU": "1.0.0", + "Commands.Common": "", + "Commands.Common.Authentication": "", + "Commands.Common.Storage": "", + "Commands.ResourceManager.Common": "", + "Microsoft.IdentityModel.Clients.ActiveDirectory": "3.6.212041202-alpha", + "Microsoft.Rest.ClientRuntime": "1.8.1", + "Newtonsoft.Json": "7.0.1", + "System.Collections": "4.0.11-beta-23516", + "System.Collections.Concurrent": "4.0.11-beta-23516", + "System.Diagnostics.TraceSource": "4.0.0-beta-23516", + "System.Diagnostics.Tracing": "4.0.21-beta-23516", + "System.IO": "4.0.11-beta-23516", + "System.IO.FileSystem": "4.0.1-beta-23516", + "System.Net.Http": "4.0.1-beta-23516", + "System.Net.WebHeaderCollection": "4.0.1-beta-23516", + "System.Reflection": "4.1.0-beta-23516", + "System.Reflection.Extensions": "4.0.1-beta-23516", + "System.Reflection.Primitives": "4.0.1-beta-23516", + "System.Reflection.TypeExtensions": "4.1.0-beta-23516", + "System.Runtime": "4.0.21-beta-23516", + "System.Runtime.Extensions": "4.0.11-beta-23516", + "System.Runtime.Serialization.Primitives": "4.1.0-beta-23516", + "System.Runtime.Serialization.Json": "4.0.1-beta-23516", + "System.Runtime.Serialization.Xml": "4.1.0-beta-23516", + "System.Security.Cryptography.Algorithms": "4.0.0-beta-23516", + "System.Security.Cryptography.X509Certificates": "4.0.0-beta-23516", + "System.Text.Encoding": "4.0.11-beta-23516", + "System.Text.Encoding.Extensions": "4.0.11-beta-23516", + "System.Threading": "4.0.11-beta-23516", + "System.Threading.Tasks": "4.0.11-beta-23516", + "System.Threading.Thread": "4.0.0-beta-23516", + "System.Xml.XmlDocument": "4.0.1-beta-23516", + "System.Xml.XPath.XmlDocument": "4.0.1-beta-23516", + "WindowsAzure.Storage": "6.1.1-preview" + }, + "compilationOptions": { "emitEntryPoint": true } +}