Cleanup in AzCopyCore Sample (#2123)

* cleaned up logging

* improved Key.ComputeKeyBytes

* cleanup AzCopyCore sample

corefxlab.sln

@@ -993,4 +993,4 @@ Global
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {9DD4022C-A010-4A9B-BCC5-171566D4CB17}
 	EndGlobalSection
-EndGlobal
+EndGlobal

samples/AzCopyCore/AzCopyCore/AzCopyCore.csproj

@@ -23,6 +23,7 @@
     <PackageReference Include="System.Memory" Version="4.5.0-preview2-26213-06" />
     <PackageReference Include="System.Text.Http" Version="0.1.0-preview2-180213-4" />
     <PackageReference Include="System.Text.Http.Parser" Version="0.1.0-preview2-180213-4" />
+    <PackageReference Include="System.Text.Utf8String" Version="0.1.0-preview2-180214-3" />
     <PackageReference Include="System.Threading.Tasks.Extensions" Version="4.5.0-preview2-26213-06" />
   </ItemGroup>
 

samples/AzCopyCore/AzCopyCore/Helpers/CommandLine.cs

@@ -0,0 +1,45 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+namespace System.CommandLine
+{
+    // TODO (pri 3): Should I use the command line library?
+    class CommandLine
+    {
+        readonly string[] _options;
+
+        public CommandLine(string[] options)
+        {
+            _options = options;
+        }
+
+        public bool Contains(string optionName)
+        {
+            for (int i = 0; i < _options.Length; i++)
+            {
+                var candidate = _options[i];
+                if (candidate.StartsWith(optionName)) return true;
+            }
+            return false;
+        }
+
+        public ReadOnlySpan<char> this[string optionName] => Get(optionName).Span;
+
+        public ReadOnlyMemory<char> Get(string optionName)
+        {
+            if (optionName.Length < 1) throw new ArgumentOutOfRangeException(nameof(optionName));
+
+            for (int i = 0; i < _options.Length; i++)
+            {
+                var candidate = _options[i];
+                if (candidate.StartsWith(optionName))
+                {
+                    var option = candidate.AsReadOnlyMemory();
+                    return option.Slice(optionName.Length);
+                }
+            }
+            return ReadOnlyMemory<char>.Empty;
+        }
+    }
+}

samples/AzCopyCore/AzCopyCore/Helpers/ConsoleTraceListener.cs

@@ -0,0 +1,67 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+namespace System.Diagnostics
+{
+    class ConsoleTraceListener : TraceListener
+    {
+        public override void Write(string message)
+            => Console.Write(message);
+
+        public override void WriteLine(string message)
+            => Console.WriteLine(message);
+
+        public override void Fail(string message)
+        {
+            base.Fail(message);
+        }
+        public override void Fail(string message, string detailMessage)
+        {
+            base.Fail(message, detailMessage);
+        }
+        public override bool IsThreadSafe => false;
+
+        public override void TraceData(TraceEventCache eventCache, string source, TraceEventType eventType, int id, params object[] data)
+        {
+            ConsoleColor color = default;
+            if (eventType == TraceEventType.Error || eventType == TraceEventType.Critical)
+            {
+                color = Console.ForegroundColor;
+                Console.ForegroundColor = ConsoleColor.Red;
+            }
+
+            Console.WriteLine(eventType.ToString());
+
+            if (eventType == TraceEventType.Error || eventType == TraceEventType.Critical)
+            {
+                Console.ForegroundColor = color;
+            }
+
+            foreach (var item in data)
+            {
+                Console.Write("\t");
+                Console.WriteLine(data);
+            }
+        }
+
+        public override void TraceEvent(TraceEventCache eventCache, string source, TraceEventType eventType, int id, string format, params object[] args)
+        {
+            ConsoleColor color = default;
+            if (eventType == TraceEventType.Error || eventType == TraceEventType.Critical)
+            {
+                color = Console.ForegroundColor;
+                Console.ForegroundColor = ConsoleColor.Red;
+            }
+
+            Console.WriteLine(format, args);
+
+            if (eventType == TraceEventType.Error || eventType == TraceEventType.Critical)
+            {
+                Console.ForegroundColor = color;
+            }
+        }
+    }
+}
+
+

samples/AzCopyCore/AzCopyCore/Helpers/PipelinesExtensions.cs

@@ -0,0 +1,63 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Buffers;
+using System.Threading.Tasks;
+
+namespace System.IO.Pipelines
+{
+    // TODO (pri 3): Would be nice to add to the platform (but NetStandard does not support the stream APIs)
+    static class PipelinesExtensions
+    {
+        /// <summary>
+        /// Copies bytes from ReadOnlySequence to a Stream
+        /// </summary>
+        public static async Task WriteAsync(this Stream stream, ReadOnlySequence<byte> buffer)
+        {
+            for (var position = buffer.Start; buffer.TryGet(ref position, out var memory);)
+            {
+                await stream.WriteAsync(memory).ConfigureAwait(false);
+            }
+        }
+
+        /// <summary>
+        /// Copies bytes from PipeReader to a Stream
+        /// </summary>
+        public static async Task WriteAsync(this Stream stream, PipeReader reader, ulong bytes)
+        {
+            while (bytes > 0)
+            {
+                var result = await reader.ReadAsync();
+                ReadOnlySequence<byte> bodyBuffer = result.Buffer;
+                if (bytes < (ulong)bodyBuffer.Length)
+                {
+                    throw new NotImplementedException();
+                }
+                bytes -= (ulong)bodyBuffer.Length;
+                await stream.WriteAsync(bodyBuffer).ConfigureAwait(false);
+                await stream.FlushAsync().ConfigureAwait(false);
+                reader.AdvanceTo(bodyBuffer.End);
+            }
+        }
+
+        /// <summary>
+        /// Copies bytes from Stream to PipeWriter 
+        /// </summary>
+        public static async Task WriteAsync(this PipeWriter writer, Stream stream)
+        {
+            if (!stream.CanRead) throw new ArgumentException("Stream.CanRead returned false", nameof(stream));
+            while (true)
+            {
+                var buffer = writer.GetMemory();
+                if (buffer.Length == 0) throw new NotSupportedException("PipeWriter.GetMemory returned an empty buffer.");
+                int read = await stream.ReadAsync(buffer).ConfigureAwait(false);
+                if (read == 0) return;
+                writer.Advance(read);
+                await writer.FlushAsync();
+            }
+        }
+    }
+}
+
+

samples/AzCopyCore/AzCopyCore/Program.cs

@@ -1,10 +1,16 @@
-using System;
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
 using System.Azure.Authentication;
 using System.Azure.Storage;
 using System.Azure.Storage.Requests;
 using System.Buffers;
+using System.CommandLine;
 using System.Diagnostics;
 using System.IO;
+using System.IO.Pipelines;
 using System.Threading.Tasks;
 
 static class Program
@@ -21,10 +27,10 @@ static void PrintUsage()
 
     static void Main(string[] args)
     {
-        Log.Listeners.Add(new TextWriterTraceListener(Console.Out));
+        Log.Listeners.Add(new ConsoleTraceListener());
         Log.Switch.Level = SourceLevels.Error;
 
-        var options = new CommandOptions(args);
+        var options = new CommandLine(args);
         ReadOnlyMemory<char> source = options.Get("/Source:");
         ReadOnlyMemory<char> destination = options.Get("/Dest:");
 
@@ -47,7 +53,7 @@ static void Main(string[] args)
         }
     }
 
-    static void TransferDirectoryToStorage(ReadOnlyMemory<char> localDirectory, ReadOnlyMemory<char> storageDirectory, CommandOptions options)
+    static void TransferDirectoryToStorage(ReadOnlyMemory<char> localDirectory, ReadOnlyMemory<char> storageDirectory, CommandLine options)
     {
         var directoryPath = new string(localDirectory.Span);     
         if (!Directory.Exists(directoryPath))
@@ -61,15 +67,13 @@ static void TransferDirectoryToStorage(ReadOnlyMemory<char> localDirectory, Read
             return;
         }
 
-        ReadOnlyMemory<char> key = options.Get("/DestKey:");
-        byte[] keyBytes = key.Span.ComputeKeyBytes();
-
-        ReadOnlyMemory<char> storageFullPath = storageDirectory.Slice(7);
+        ReadOnlyMemory<char> storageFullPath = storageDirectory.Slice("http://".Length);
         int pathStart = storageFullPath.Span.IndexOf('/');
         ReadOnlyMemory<char> host = storageFullPath.Slice(0, pathStart);
         ReadOnlyMemory<char> path = storageFullPath.Slice(pathStart + 1);
         ReadOnlyMemory<char> account = storageFullPath.Slice(0, storageFullPath.Span.IndexOf('.'));
 
+        byte[] keyBytes = options["/DestKey:"].ComputeKeyBytes();
         using (var client = new StorageClient(keyBytes, account, host))
         {
             client.Log = Log;           
@@ -87,7 +91,7 @@ static void TransferDirectoryToStorage(ReadOnlyMemory<char> localDirectory, Read
         }
     }
 
-    static void TransferStorageFileToDirectory(ReadOnlyMemory<char> storageFile, ReadOnlyMemory<char> localDirectory, CommandOptions options)
+    static void TransferStorageFileToDirectory(ReadOnlyMemory<char> storageFile, ReadOnlyMemory<char> localDirectory, CommandLine options)
     {
         var directory = new string(localDirectory.Span);
         if (!options.Contains("/SourceKey:"))
@@ -96,10 +100,7 @@ static void TransferStorageFileToDirectory(ReadOnlyMemory<char> storageFile, Rea
             return;
         }
 
-        ReadOnlyMemory<char> key = options.Get("/SourceKey:");
-        byte[] keyBytes = key.Span.ComputeKeyBytes();
-
-        ReadOnlyMemory<char> storageFullPath = storageFile.Slice(7);
+        ReadOnlyMemory<char> storageFullPath = storageFile.Slice("http://".Length);
         int pathStart = storageFullPath.Span.IndexOf('/');
         ReadOnlyMemory<char> host = storageFullPath.Slice(0, pathStart);
         ReadOnlyMemory<char> storagePath = storageFullPath.Slice(pathStart + 1);
@@ -112,6 +113,7 @@ static void TransferStorageFileToDirectory(ReadOnlyMemory<char> storageFile, Rea
             Directory.CreateDirectory(directory);
         }
 
+        byte[] keyBytes = options["/SourceKey:"].ComputeKeyBytes();
         string destinationPath = directory + "\\" + new string(file.Span);
         using (var client = new StorageClient(keyBytes, account, host))
         {
@@ -140,7 +142,7 @@ static async ValueTask<bool> CopyLocalFileToStorageFile(StorageClient client, st
             }
         }
 
-        Log.WriteError($"Response Error {response.StatusCode}");
+        Log.TraceEvent(TraceEventType.Error, 0, "Response Status Code {0}", response.StatusCode);
         return false;
     }
 
@@ -151,7 +153,7 @@ static async ValueTask<bool> CopyStorageFileToLocalFile(StorageClient client, st
 
         if (response.StatusCode != 200)
         {
-            Log.WriteError($"Response Error {response.StatusCode}");
+            Log.TraceEvent(TraceEventType.Error, 0, "Response Status Code {0}", response.StatusCode);
             return false;
         }
 

samples/AzCopyCore/AzCopyCore/SocketClient.cs

@@ -1,10 +1,15 @@
-using System.Buffers;
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Buffers;
 using System.Diagnostics;
 using System.IO;
 using System.IO.Pipelines;
 using System.Net.Security;
 using System.Net.Sockets;
 using System.Text.Http.Parser;
+using System.Text.Utf8;
 using System.Threading.Tasks;
 
 // SocketClient is an experimental low-allocating/low-copy HTTP client API
@@ -100,7 +105,7 @@ public static async ValueTask<T> ParseAsync<T>(PipeReader reader, TraceSource lo
             var result = await reader.ReadAsync();
             ReadOnlySequence<byte> buffer = result.Buffer;
 
-            if (log != null) log.WriteInformation("RESPONSE: ", buffer.First);
+            if (log != null) log.TraceInformation("RESPONSE:\n{0}", new Utf8String(buffer.First.Span));
 
             var handler = new T();
             // TODO (pri 2): this should not be static, or all should be static
@@ -152,7 +157,7 @@ async Task SendAsync()
             }
             catch(Exception e)
             {
-                Log.WriteError(e.ToString());
+                Log.TraceEvent(TraceEventType.Error, 0, e.ToString());
             }
             finally
             {
@@ -176,7 +181,7 @@ async Task ReceiveAsync()
                         var readBytes = await ReadFromSocketAsync(buffer).ConfigureAwait(false);
                         if (readBytes == 0) break;
 
-                        if (Log != null) Log.WriteInformation($"RESPONSE {readBytes}", buffer.Slice(0, readBytes));
+                        if (Log != null) Log.TraceInformation(new Utf8String(buffer.Span.Slice(0, readBytes)).ToString());
 
                         writer.Advance(readBytes);
                         await writer.FlushAsync();

samples/AzCopyCore/AzCopyCore/StorageClient.cs

@@ -1,4 +1,8 @@
-using System.Azure.Authentication;
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Azure.Authentication;
 using System.Azure.Storage.Requests;
 using System.Buffers.Cryptography;
 using System.Buffers.Text;

samples/AzCopyCore/AzCopyCore/StorageRequests.cs

@@ -1,5 +1,8 @@
-using System.Azure.Authentication;
-using System.Buffers;
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Azure.Authentication;
 using System.Buffers.Text;
 using System.Buffers.Transformations;
 using System.IO;