diff --git a/corefxlab.sln b/corefxlab.sln index de1eb1b7374..02b77eb41b9 100644 --- a/corefxlab.sln +++ b/corefxlab.sln @@ -993,4 +993,4 @@ Global GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {9DD4022C-A010-4A9B-BCC5-171566D4CB17} EndGlobalSection -EndGlobal +EndGlobal \ No newline at end of file diff --git a/samples/AzCopyCore/AzCopyCore/AzCopyCore.csproj b/samples/AzCopyCore/AzCopyCore/AzCopyCore.csproj index 6567f2197ab..4b7f1684a56 100644 --- a/samples/AzCopyCore/AzCopyCore/AzCopyCore.csproj +++ b/samples/AzCopyCore/AzCopyCore/AzCopyCore.csproj @@ -23,6 +23,7 @@ + diff --git a/samples/AzCopyCore/AzCopyCore/Helpers/CommandLine.cs b/samples/AzCopyCore/AzCopyCore/Helpers/CommandLine.cs new file mode 100644 index 00000000000..ac83f21bef6 --- /dev/null +++ b/samples/AzCopyCore/AzCopyCore/Helpers/CommandLine.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace System.CommandLine +{ + // TODO (pri 3): Should I use the command line library? + class CommandLine + { + readonly string[] _options; + + public CommandLine(string[] options) + { + _options = options; + } + + public bool Contains(string optionName) + { + for (int i = 0; i < _options.Length; i++) + { + var candidate = _options[i]; + if (candidate.StartsWith(optionName)) return true; + } + return false; + } + + public ReadOnlySpan this[string optionName] => Get(optionName).Span; + + public ReadOnlyMemory Get(string optionName) + { + if (optionName.Length < 1) throw new ArgumentOutOfRangeException(nameof(optionName)); + + for (int i = 0; i < _options.Length; i++) + { + var candidate = _options[i]; + if (candidate.StartsWith(optionName)) + { + var option = candidate.AsReadOnlyMemory(); + return option.Slice(optionName.Length); + } + } + return ReadOnlyMemory.Empty; + } + } +} diff --git a/samples/AzCopyCore/AzCopyCore/Helpers/ConsoleTraceListener.cs b/samples/AzCopyCore/AzCopyCore/Helpers/ConsoleTraceListener.cs new file mode 100644 index 00000000000..37d29b72cc7 --- /dev/null +++ b/samples/AzCopyCore/AzCopyCore/Helpers/ConsoleTraceListener.cs @@ -0,0 +1,67 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace System.Diagnostics +{ + class ConsoleTraceListener : TraceListener + { + public override void Write(string message) + => Console.Write(message); + + public override void WriteLine(string message) + => Console.WriteLine(message); + + public override void Fail(string message) + { + base.Fail(message); + } + public override void Fail(string message, string detailMessage) + { + base.Fail(message, detailMessage); + } + public override bool IsThreadSafe => false; + + public override void TraceData(TraceEventCache eventCache, string source, TraceEventType eventType, int id, params object[] data) + { + ConsoleColor color = default; + if (eventType == TraceEventType.Error || eventType == TraceEventType.Critical) + { + color = Console.ForegroundColor; + Console.ForegroundColor = ConsoleColor.Red; + } + + Console.WriteLine(eventType.ToString()); + + if (eventType == TraceEventType.Error || eventType == TraceEventType.Critical) + { + Console.ForegroundColor = color; + } + + foreach (var item in data) + { + Console.Write("\t"); + Console.WriteLine(data); + } + } + + public override void TraceEvent(TraceEventCache eventCache, string source, TraceEventType eventType, int id, string format, params object[] args) + { + ConsoleColor color = default; + if (eventType == TraceEventType.Error || eventType == TraceEventType.Critical) + { + color = Console.ForegroundColor; + Console.ForegroundColor = ConsoleColor.Red; + } + + Console.WriteLine(format, args); + + if (eventType == TraceEventType.Error || eventType == TraceEventType.Critical) + { + Console.ForegroundColor = color; + } + } + } +} + + diff --git a/samples/AzCopyCore/AzCopyCore/Helpers/PipelinesExtensions.cs b/samples/AzCopyCore/AzCopyCore/Helpers/PipelinesExtensions.cs new file mode 100644 index 00000000000..e974906a976 --- /dev/null +++ b/samples/AzCopyCore/AzCopyCore/Helpers/PipelinesExtensions.cs @@ -0,0 +1,63 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Buffers; +using System.Threading.Tasks; + +namespace System.IO.Pipelines +{ + // TODO (pri 3): Would be nice to add to the platform (but NetStandard does not support the stream APIs) + static class PipelinesExtensions + { + /// + /// Copies bytes from ReadOnlySequence to a Stream + /// + public static async Task WriteAsync(this Stream stream, ReadOnlySequence buffer) + { + for (var position = buffer.Start; buffer.TryGet(ref position, out var memory);) + { + await stream.WriteAsync(memory).ConfigureAwait(false); + } + } + + /// + /// Copies bytes from PipeReader to a Stream + /// + public static async Task WriteAsync(this Stream stream, PipeReader reader, ulong bytes) + { + while (bytes > 0) + { + var result = await reader.ReadAsync(); + ReadOnlySequence bodyBuffer = result.Buffer; + if (bytes < (ulong)bodyBuffer.Length) + { + throw new NotImplementedException(); + } + bytes -= (ulong)bodyBuffer.Length; + await stream.WriteAsync(bodyBuffer).ConfigureAwait(false); + await stream.FlushAsync().ConfigureAwait(false); + reader.AdvanceTo(bodyBuffer.End); + } + } + + /// + /// Copies bytes from Stream to PipeWriter + /// + public static async Task WriteAsync(this PipeWriter writer, Stream stream) + { + if (!stream.CanRead) throw new ArgumentException("Stream.CanRead returned false", nameof(stream)); + while (true) + { + var buffer = writer.GetMemory(); + if (buffer.Length == 0) throw new NotSupportedException("PipeWriter.GetMemory returned an empty buffer."); + int read = await stream.ReadAsync(buffer).ConfigureAwait(false); + if (read == 0) return; + writer.Advance(read); + await writer.FlushAsync(); + } + } + } +} + + diff --git a/samples/AzCopyCore/AzCopyCore/Program.cs b/samples/AzCopyCore/AzCopyCore/Program.cs index 87d95aa8ffd..8791342483f 100644 --- a/samples/AzCopyCore/AzCopyCore/Program.cs +++ b/samples/AzCopyCore/AzCopyCore/Program.cs @@ -1,10 +1,16 @@ -using System; +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; using System.Azure.Authentication; using System.Azure.Storage; using System.Azure.Storage.Requests; using System.Buffers; +using System.CommandLine; using System.Diagnostics; using System.IO; +using System.IO.Pipelines; using System.Threading.Tasks; static class Program @@ -21,10 +27,10 @@ static void PrintUsage() static void Main(string[] args) { - Log.Listeners.Add(new TextWriterTraceListener(Console.Out)); + Log.Listeners.Add(new ConsoleTraceListener()); Log.Switch.Level = SourceLevels.Error; - var options = new CommandOptions(args); + var options = new CommandLine(args); ReadOnlyMemory source = options.Get("/Source:"); ReadOnlyMemory destination = options.Get("/Dest:"); @@ -47,7 +53,7 @@ static void Main(string[] args) } } - static void TransferDirectoryToStorage(ReadOnlyMemory localDirectory, ReadOnlyMemory storageDirectory, CommandOptions options) + static void TransferDirectoryToStorage(ReadOnlyMemory localDirectory, ReadOnlyMemory storageDirectory, CommandLine options) { var directoryPath = new string(localDirectory.Span); if (!Directory.Exists(directoryPath)) @@ -61,15 +67,13 @@ static void TransferDirectoryToStorage(ReadOnlyMemory localDirectory, Read return; } - ReadOnlyMemory key = options.Get("/DestKey:"); - byte[] keyBytes = key.Span.ComputeKeyBytes(); - - ReadOnlyMemory storageFullPath = storageDirectory.Slice(7); + ReadOnlyMemory storageFullPath = storageDirectory.Slice("http://".Length); int pathStart = storageFullPath.Span.IndexOf('/'); ReadOnlyMemory host = storageFullPath.Slice(0, pathStart); ReadOnlyMemory path = storageFullPath.Slice(pathStart + 1); ReadOnlyMemory account = storageFullPath.Slice(0, storageFullPath.Span.IndexOf('.')); + byte[] keyBytes = options["/DestKey:"].ComputeKeyBytes(); using (var client = new StorageClient(keyBytes, account, host)) { client.Log = Log; @@ -87,7 +91,7 @@ static void TransferDirectoryToStorage(ReadOnlyMemory localDirectory, Read } } - static void TransferStorageFileToDirectory(ReadOnlyMemory storageFile, ReadOnlyMemory localDirectory, CommandOptions options) + static void TransferStorageFileToDirectory(ReadOnlyMemory storageFile, ReadOnlyMemory localDirectory, CommandLine options) { var directory = new string(localDirectory.Span); if (!options.Contains("/SourceKey:")) @@ -96,10 +100,7 @@ static void TransferStorageFileToDirectory(ReadOnlyMemory storageFile, Rea return; } - ReadOnlyMemory key = options.Get("/SourceKey:"); - byte[] keyBytes = key.Span.ComputeKeyBytes(); - - ReadOnlyMemory storageFullPath = storageFile.Slice(7); + ReadOnlyMemory storageFullPath = storageFile.Slice("http://".Length); int pathStart = storageFullPath.Span.IndexOf('/'); ReadOnlyMemory host = storageFullPath.Slice(0, pathStart); ReadOnlyMemory storagePath = storageFullPath.Slice(pathStart + 1); @@ -112,6 +113,7 @@ static void TransferStorageFileToDirectory(ReadOnlyMemory storageFile, Rea Directory.CreateDirectory(directory); } + byte[] keyBytes = options["/SourceKey:"].ComputeKeyBytes(); string destinationPath = directory + "\\" + new string(file.Span); using (var client = new StorageClient(keyBytes, account, host)) { @@ -140,7 +142,7 @@ static async ValueTask CopyLocalFileToStorageFile(StorageClient client, st } } - Log.WriteError($"Response Error {response.StatusCode}"); + Log.TraceEvent(TraceEventType.Error, 0, "Response Status Code {0}", response.StatusCode); return false; } @@ -151,7 +153,7 @@ static async ValueTask CopyStorageFileToLocalFile(StorageClient client, st if (response.StatusCode != 200) { - Log.WriteError($"Response Error {response.StatusCode}"); + Log.TraceEvent(TraceEventType.Error, 0, "Response Status Code {0}", response.StatusCode); return false; } diff --git a/samples/AzCopyCore/AzCopyCore/SocketClient.cs b/samples/AzCopyCore/AzCopyCore/SocketClient.cs index 6b6e3de803f..6de6670f1a3 100644 --- a/samples/AzCopyCore/AzCopyCore/SocketClient.cs +++ b/samples/AzCopyCore/AzCopyCore/SocketClient.cs @@ -1,10 +1,15 @@ -using System.Buffers; +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Buffers; using System.Diagnostics; using System.IO; using System.IO.Pipelines; using System.Net.Security; using System.Net.Sockets; using System.Text.Http.Parser; +using System.Text.Utf8; using System.Threading.Tasks; // SocketClient is an experimental low-allocating/low-copy HTTP client API @@ -100,7 +105,7 @@ public static async ValueTask ParseAsync(PipeReader reader, TraceSource lo var result = await reader.ReadAsync(); ReadOnlySequence buffer = result.Buffer; - if (log != null) log.WriteInformation("RESPONSE: ", buffer.First); + if (log != null) log.TraceInformation("RESPONSE:\n{0}", new Utf8String(buffer.First.Span)); var handler = new T(); // TODO (pri 2): this should not be static, or all should be static @@ -152,7 +157,7 @@ async Task SendAsync() } catch(Exception e) { - Log.WriteError(e.ToString()); + Log.TraceEvent(TraceEventType.Error, 0, e.ToString()); } finally { @@ -176,7 +181,7 @@ async Task ReceiveAsync() var readBytes = await ReadFromSocketAsync(buffer).ConfigureAwait(false); if (readBytes == 0) break; - if (Log != null) Log.WriteInformation($"RESPONSE {readBytes}", buffer.Slice(0, readBytes)); + if (Log != null) Log.TraceInformation(new Utf8String(buffer.Span.Slice(0, readBytes)).ToString()); writer.Advance(readBytes); await writer.FlushAsync(); diff --git a/samples/AzCopyCore/AzCopyCore/StorageClient.cs b/samples/AzCopyCore/AzCopyCore/StorageClient.cs index a92b7ce4b48..5877ad71e07 100644 --- a/samples/AzCopyCore/AzCopyCore/StorageClient.cs +++ b/samples/AzCopyCore/AzCopyCore/StorageClient.cs @@ -1,4 +1,8 @@ -using System.Azure.Authentication; +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Azure.Authentication; using System.Azure.Storage.Requests; using System.Buffers.Cryptography; using System.Buffers.Text; diff --git a/samples/AzCopyCore/AzCopyCore/StorageRequests.cs b/samples/AzCopyCore/AzCopyCore/StorageRequests.cs index c0b9ec651ec..06e43ba15bb 100644 --- a/samples/AzCopyCore/AzCopyCore/StorageRequests.cs +++ b/samples/AzCopyCore/AzCopyCore/StorageRequests.cs @@ -1,5 +1,8 @@ -using System.Azure.Authentication; -using System.Buffers; +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Azure.Authentication; using System.Buffers.Text; using System.Buffers.Transformations; using System.IO; diff --git a/samples/AzCopyCore/AzCopyCore/temp/Extensions.cs b/samples/AzCopyCore/AzCopyCore/temp/Extensions.cs deleted file mode 100644 index f3f6042b29e..00000000000 --- a/samples/AzCopyCore/AzCopyCore/temp/Extensions.cs +++ /dev/null @@ -1,122 +0,0 @@ -using System.Buffers.Text; -using System.Diagnostics; -using System.IO; -using System.IO.Pipelines; -using System.Text.Http.Parser; -using System.Threading.Tasks; - -namespace System.Buffers -{ - // TODO (pri 3): Add to the platform (but NetStandard does not support the stream APIs) - static class GeneralExtensions - { - /// - /// Copies bytes from ReadOnlySequence to a Stream - /// - public static async Task WriteAsync(this Stream stream, ReadOnlySequence buffer) - { - for (var position = buffer.Start; buffer.TryGet(ref position, out var memory);) - { - await stream.WriteAsync(memory).ConfigureAwait(false); - } - } - - /// - /// Copies bytes from PipeReader to a Stream - /// - public static async Task WriteAsync(this Stream stream, PipeReader reader, ulong bytes) - { - while (bytes > 0) - { - var result = await reader.ReadAsync(); - ReadOnlySequence bodyBuffer = result.Buffer; - if (bytes < (ulong)bodyBuffer.Length) - { - throw new NotImplementedException(); - } - bytes -= (ulong)bodyBuffer.Length; - await stream.WriteAsync(bodyBuffer).ConfigureAwait(false); - await stream.FlushAsync().ConfigureAwait(false); - reader.AdvanceTo(bodyBuffer.End); - } - } - - /// - /// Copies bytes from Stream to PipeWriter - /// - public static async Task WriteAsync(this PipeWriter writer, Stream stream) - { - if (!stream.CanRead) throw new ArgumentException("Stream.CanRead returned false", nameof(stream)); - while (true) - { - var buffer = writer.GetMemory(); - if (buffer.Length == 0) throw new NotSupportedException("PipeWriter.GetMemory returned an empty buffer."); - int read = await stream.ReadAsync(buffer).ConfigureAwait(false); - if (read == 0) return; - writer.Advance(read); - await writer.FlushAsync(); - } - } - } - - // TODO (pri 3): Is TraceSource the right logger here? - public static class TraceListenerExtensions - { - public static void WriteInformation(this TraceSource source, string tag, ReadOnlyMemory utf8Text) - { - if (source.Switch.ShouldTrace(TraceEventType.Information)) - { - var message = Encodings.Utf8.ToString(utf8Text.Span); - source.TraceInformation($"{tag}:\n{message}\n"); - } - } - - public static void WriteError(this TraceSource source, string message) - { - if (source.Switch.ShouldTrace(TraceEventType.Error)) - { - var color = Console.ForegroundColor; - Console.ForegroundColor = ConsoleColor.Red; - source.TraceEvent(TraceEventType.Error, 0, message); - Console.ForegroundColor = color; - } - } - } - - // TODO (pri 3): Should I use the command line library? - class CommandOptions - { - readonly string[] _options; - - public CommandOptions(string[] options) - { - _options = options; - } - - public bool Contains(string optionName) - { - for (int i = 0; i < _options.Length; i++) - { - var candidate = _options[i]; - if (candidate.StartsWith(optionName)) return true; - } - return false; - } - - public ReadOnlyMemory Get(string optionName) - { - if (optionName.Length < 1) throw new ArgumentOutOfRangeException(nameof(optionName)); - - for (int i = 0; i < _options.Length; i++) - { - var candidate = _options[i]; - if (candidate.StartsWith(optionName)) - { - var option = candidate.AsReadOnlyMemory(); - return option.Slice(optionName.Length); - } - } - return ReadOnlyMemory.Empty; - } - } -} diff --git a/src/System.Azure.Experimental/System/Azure/Key.cs b/src/System.Azure.Experimental/System/Azure/Key.cs index fa02a3f29cc..3cb51489ebc 100644 --- a/src/System.Azure.Experimental/System/Azure/Key.cs +++ b/src/System.Azure.Experimental/System/Azure/Key.cs @@ -8,41 +8,52 @@ namespace System.Azure.Authentication { public static class Key { public static byte[] ComputeKeyBytes(string key) + => ComputeKeyBytes(key.AsReadOnlySpan()); + + public static byte[] ComputeKeyBytes(this ReadOnlySpan key) { - int size = key.Length * 2; + var utf16Bytes = key.AsBytes(); + int size = utf16Bytes.Length; // the input must be ASCII (i.e. Base64 encoded) + var buffer = size < 128 ? stackalloc byte[size] : new byte[size]; - if (Encodings.Utf16.ToUtf8(key.AsReadOnlySpan().AsBytes(), buffer, out int consumed, out int written) != OperationStatus.Done) + var result = Encodings.Utf16.ToUtf8(utf16Bytes, buffer, out int consumed, out int written); + if (result != OperationStatus.Done) { - throw new NotImplementedException("need to resize buffer"); + throw new ArgumentOutOfRangeException(nameof(key), $"ToUtf8 returned {result}"); } var keyBytes = new byte[64]; - var result = Base64.DecodeFromUtf8(buffer.Slice(0, written), keyBytes, out consumed, out written); - if (result != OperationStatus.Done || written != 64) + result = Base64.DecodeFromUtf8(buffer.Slice(0, written), keyBytes, out consumed, out written); + if (result != OperationStatus.Done) { - throw new NotImplementedException("need to resize buffer"); + throw new ArgumentOutOfRangeException(nameof(key), $"Base64.Decode returned {result}"); + } + if (written != 64) + { + throw new ArgumentOutOfRangeException(nameof(key), $"{written}!={64}"); } return keyBytes; } - public static byte[] ComputeKeyBytes(this ReadOnlySpan key) + public static bool TryComputeKeyBytes(this ReadOnlySpan key, Span keyBytes) { - int size = key.Length * 2; + var utf16Bytes = key.AsBytes(); + int size = utf16Bytes.Length; // the input must be ASCII (i.e. Base64 encoded) + var buffer = size < 128 ? stackalloc byte[size] : new byte[size]; - if (Encodings.Utf16.ToUtf8(key.AsBytes(), buffer, out int consumed, out int written) != OperationStatus.Done) + var result = Encodings.Utf16.ToUtf8(utf16Bytes, buffer, out int consumed, out int written); + if (result != OperationStatus.Done) { - throw new NotImplementedException("need to resize buffer"); + throw new ArgumentOutOfRangeException(nameof(key), $"ToUtf8 returned {result}"); } - var keyBytes = new byte[64]; - var result = Base64.DecodeFromUtf8(buffer.Slice(0, written), keyBytes, out consumed, out written); - if (result != OperationStatus.Done || written != 64) - { - throw new NotImplementedException("need to resize buffer"); - } - return keyBytes; + result = Base64.DecodeFromUtf8(buffer.Slice(0, written), keyBytes, out consumed, out written); + if (result == OperationStatus.Done) return true; + if (result == OperationStatus.DestinationTooSmall) return false; + + throw new ArgumentOutOfRangeException(nameof(key), $"Base64.Decode returned {result}"); } } }