Skip to content
This repository has been archived by the owner on Aug 2, 2023. It is now read-only.

Updated AzCopyCore sample to new corfxlab APIs #2122

Merged
merged 1 commit into from
Feb 14, 2018
Merged

Updated AzCopyCore sample to new corfxlab APIs #2122

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions samples/AzCopyCore/AzCopyCore/AzCopyCore.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,13 @@
</ItemGroup>

<ItemGroup>
<PackageReference Include="System.Azure.Experimental" Version="0.1.0-preview2-180209-2" />
<PackageReference Include="System.Buffers.Experimental" Version="0.1.0-preview2-180209-2" />
<PackageReference Include="System.IO.Pipelines" Version="0.1.0-preview2-180209-2" />
<PackageReference Include="System.Memory" Version="4.5.0-preview2-26209-05" />
<PackageReference Include="System.Text.Http" Version="0.1.0-preview2-180209-2" />
<PackageReference Include="System.Text.Http.Parser" Version="0.1.0-preview2-180209-2" />
<PackageReference Include="System.Threading.Tasks.Extensions" Version="4.5.0-preview2-26209-05" />
<PackageReference Include="System.Azure.Experimental" Version="0.1.0-preview2-180213-4" />
Copy link
Member

@ahsonkhan ahsonkhan Feb 14, 2018 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Consider adding it to the solution and reference the libraries directly (we have some samples which are written this way already). This would remove the need to keep updating these versions.

Also consider importing common.props and remove the RuntimeFrameworkVersion attribute.

See the following as an example:
https://github.com/dotnet/corefxlab/blob/master/samples/QotdService/QotdService.csproj

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I kind like the samples being standalone apps as opposed to having to build them with the whole repo.

Copy link
Member

@ahsonkhan ahsonkhan Feb 14, 2018 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am fine with that. Should we add this standalone sample to the build script though so it gets built as part of the CI run? It would require no additional changes if it was part of the solution file. Otherwise, we can just add a "build samples" step.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, I will do it tomorrow.

<PackageReference Include="System.Buffers.Experimental" Version="0.1.0-preview2-180213-4" />
<PackageReference Include="System.IO.Pipelines" Version="0.1.0-preview2-180213-4" />
<PackageReference Include="System.Memory" Version="4.5.0-preview2-26213-06" />
<PackageReference Include="System.Text.Http" Version="0.1.0-preview2-180213-4" />
<PackageReference Include="System.Text.Http.Parser" Version="0.1.0-preview2-180213-4" />
<PackageReference Include="System.Threading.Tasks.Extensions" Version="4.5.0-preview2-26213-06" />
</ItemGroup>

</Project>
31 changes: 30 additions & 1 deletion samples/AzCopyCore/AzCopyCore/SocketClient.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,11 +86,40 @@ public async ValueTask<TResponse> SendRequest<TRequest, TResponse>(TRequest requ
await request.WriteAsync(_requestPipe.Writer).ConfigureAwait(false);

var reader = _responsePipe.Reader;
var response = await HttpExtensions.ParseAsync<TResponse>(reader, Log).ConfigureAwait(false);
var response = await ParseAsync<TResponse>(reader, Log).ConfigureAwait(false);
response.OnBody(reader);
return response;
}

static HttpParser s_headersParser = new HttpParser();

// TODO (pri 3): Add to the platform, but this would require common logging API
public static async ValueTask<T> ParseAsync<T>(PipeReader reader, TraceSource log = null)
where T : IHttpResponseLineHandler, IHttpHeadersHandler, new()
{
var result = await reader.ReadAsync();
ReadOnlySequence<byte> buffer = result.Buffer;

if (log != null) log.WriteInformation("RESPONSE: ", buffer.First);

var handler = new T();
// TODO (pri 2): this should not be static, or all should be static
if (!HttpParser.ParseResponseLine(ref handler, ref buffer, out int rlConsumed))
{
throw new NotImplementedException("could not parse the response");
}

buffer = buffer.Slice(rlConsumed);
if (!s_headersParser.ParseHeaders(ref handler, buffer, out int hdConsumed))
{
throw new NotImplementedException("could not parse the response");
}

reader.AdvanceTo(buffer.GetPosition(buffer.Start, hdConsumed));

return handler;
}

async Task SendAsync()
{
var reader = _requestPipe.Reader;
Expand Down
10 changes: 6 additions & 4 deletions samples/AzCopyCore/AzCopyCore/StorageRequests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.IO;
using System.IO.Pipelines;
using System.Net.Experimental;
using System.Text;
using System.Text.Http.Formatter;
using System.Text.Http.Parser;
using System.Threading.Tasks;
Expand Down Expand Up @@ -71,11 +72,12 @@ protected virtual void WriteOtherHeaders(ref BufferWriter writer, ref T argument
writer.WriteHeader("Host", arguments.Client.Host);
}

// TODO (pri 2): this should be UTF8
public static string AsString(Http.Method verb)
static readonly byte[] s_GETu8 = Encoding.ASCII.GetBytes("GET");
static readonly byte[] s_PUTu8 = Encoding.ASCII.GetBytes("PUT");
public static ReadOnlyMemory<byte> AsString(Http.Method verb)
{
if (verb == Http.Method.Get) return "GET";
if (verb == Http.Method.Put) return "PUT";
if (verb == Http.Method.Get) return s_GETu8;
if (verb == Http.Method.Put) return s_PUTu8;
throw new NotImplementedException();
}
}
Expand Down
105 changes: 3 additions & 102 deletions samples/AzCopyCore/AzCopyCore/temp/Extensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
using System.Diagnostics;
using System.IO;
using System.IO.Pipelines;
using System.Text;
using System.Text.Http.Parser;
using System.Threading.Tasks;

Expand All @@ -12,9 +11,9 @@ namespace System.Buffers
static class GeneralExtensions
{
/// <summary>
/// Copies bytes from ReadOnlyBuffer to a Stream
/// Copies bytes from ReadOnlySequence to a Stream
/// </summary>
public static async Task WriteAsync(this Stream stream, ReadOnlyBuffer<byte> buffer)
public static async Task WriteAsync(this Stream stream, ReadOnlySequence<byte> buffer)
{
for (var position = buffer.Start; buffer.TryGet(ref position, out var memory);)
{
Expand All @@ -30,7 +29,7 @@ public static async Task WriteAsync(this Stream stream, PipeReader reader, ulong
while (bytes > 0)
{
var result = await reader.ReadAsync();
ReadOnlyBuffer<byte> bodyBuffer = result.Buffer;
ReadOnlySequence<byte> bodyBuffer = result.Buffer;
if (bytes < (ulong)bodyBuffer.Length)
{
throw new NotImplementedException();
Expand Down Expand Up @@ -84,104 +83,6 @@ public static void WriteError(this TraceSource source, string message)
}
}

public static class HttpExtensions
{
static HttpParser s_headersParser = new HttpParser();
private const byte ByteLF = (byte)'\n';
private const byte ByteCR = (byte)'\r';
private const long maxRequestLineLength = 1024;
static readonly byte[] s_Eol = Encoding.ASCII.GetBytes("\r\n");
static readonly byte[] s_http11 = Encoding.ASCII.GetBytes("HTTP/1.1");
static readonly byte[] s_http10 = Encoding.ASCII.GetBytes("HTTP/1.0");
static readonly byte[] s_http20 = Encoding.ASCII.GetBytes("HTTP/2.0");

// TODO (pri 2): move to corfxlab
public static bool ParseResponseLine<T>(ref T handler, ref ReadOnlyBuffer<byte> buffer, out int consumedBytes) where T : IHttpResponseLineHandler
{
var line = buffer.First.Span;
var lf = line.IndexOf(ByteLF);
if (lf >= 0)
{
line = line.Slice(0, lf + 1);
}
else if (buffer.IsSingleSegment)
{
consumedBytes = 0;
return false;
}
else
{
long index = Sequence.IndexOf(buffer, ByteLF);
if(index < 0)
{
consumedBytes = 0;
return false;
}
if(index > maxRequestLineLength)
{
throw new Exception("invalid response (LF too far)");
}
line = line.Slice(0, lf + 1);
}

if(line[lf - 1] != ByteCR)
{
throw new Exception("invalid response (no CR)");
}

Http.Version version;
if (line.StartsWith(s_http11)) { version = Http.Version.Http11; }
// TODO (pri 2): add HTTP2 to HTTP.Version
else if (line.StartsWith(s_http20)) { version = Http.Version.Unknown; }
else if (line.StartsWith(s_http10)) { version = Http.Version.Http10; }
else
{
throw new Exception("invalid response (version)");
}

int codeStart = line.IndexOf((byte)' ') + 1;
var codeSlice = line.Slice(codeStart);
if (!Utf8Parser.TryParse(codeSlice, out ushort code, out consumedBytes))
{
throw new Exception("invalid response (status code)");
}

var reasonStart = consumedBytes + 1;
var reason = codeSlice.Slice(reasonStart, codeSlice.Length - reasonStart - 2);
consumedBytes = lf + s_Eol.Length;

handler.OnStatusLine(version, code, reason);

return true;
}

// TODO (pri 3): Add to the platform, but this would require common logging API
public static async ValueTask<T> ParseAsync<T>(PipeReader reader, TraceSource log = null)
where T : IHttpResponseLineHandler, IHttpHeadersHandler, new()
{
var result = await reader.ReadAsync();
ReadOnlyBuffer<byte> buffer = result.Buffer;

if (log != null) log.WriteInformation("RESPONSE: ", buffer.First);

var handler = new T();
if (!ParseResponseLine(ref handler, ref buffer, out int rlConsumed))
{
throw new NotImplementedException("could not parse the response");
}

buffer = buffer.Slice(rlConsumed);
if (!s_headersParser.ParseHeaders(ref handler, buffer, out int hdConsumed))
{
throw new NotImplementedException("could not parse the response");
}

reader.AdvanceTo(buffer.GetPosition(buffer.Start, hdConsumed));

return handler;
}
}

// TODO (pri 3): Should I use the command line library?
class CommandOptions
{
Expand Down