diff --git a/samples/AzCopyCore/AzCopyCore/Helpers/PipelinesExtensions.cs b/samples/AzCopyCore/AzCopyCore/Helpers/PipelinesExtensions.cs index eebdf674c2f..2787d5dbc4a 100644 --- a/samples/AzCopyCore/AzCopyCore/Helpers/PipelinesExtensions.cs +++ b/samples/AzCopyCore/AzCopyCore/Helpers/PipelinesExtensions.cs @@ -44,16 +44,21 @@ public static async Task WriteAsync(this Stream stream, PipeReader reader, ulong /// /// Copies bytes from Stream to PipeWriter /// - public static async Task WriteAsync(this PipeWriter writer, Stream stream) + public static async Task WriteAsync(this PipeWriter writer, Stream stream, long bytesToWrite) { if (!stream.CanRead) throw new ArgumentException("Stream.CanRead returned false", nameof(stream)); - while (true) + while (bytesToWrite > 0) { Memory buffer = writer.GetMemory(); + if (buffer.Length > bytesToWrite) + { + buffer = buffer.Slice(0, (int)bytesToWrite); + } if (buffer.Length == 0) throw new NotSupportedException("PipeWriter.GetMemory returned an empty buffer."); int read = await stream.ReadAsync(buffer).ConfigureAwait(false); if (read == 0) return; writer.Advance(read); + bytesToWrite -= read; await writer.FlushAsync(); } } diff --git a/samples/AzCopyCore/AzCopyCore/Program.cs b/samples/AzCopyCore/AzCopyCore/Program.cs index 66ee8db4490..56417daf8c4 100644 --- a/samples/AzCopyCore/AzCopyCore/Program.cs +++ b/samples/AzCopyCore/AzCopyCore/Program.cs @@ -28,7 +28,7 @@ static void PrintUsage() static void Main(string[] args) { Log.Listeners.Add(new ConsoleTraceListener()); - Log.Switch.Level = SourceLevels.Information; + Log.Switch.Level = SourceLevels.Error; var options = new CommandLine(args); ReadOnlySpan source = options.GetSpan("/Source:"); @@ -197,14 +197,27 @@ static async ValueTask CopyLocalFileToStorageFile(StorageClient client, st { using (var bytes = new FileStream(localFilePath, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true)) { - var putRequest = new PutRangeRequest(storagePath, bytes); - response = await client.SendRequest(putRequest).ConfigureAwait(false); - if (response.StatusCode == 201) return true; + long bytesLeft = bytes.Length; + long index = 0; + int length = 1024 * 1024 * 4; + while (bytesLeft > 0) + { + if (bytesLeft < length) length = (int)bytesLeft; + var putRequest = new PutRangeRequest(storagePath, bytes, index, length); + response = await client.SendRequest(putRequest).ConfigureAwait(false); + if (response.StatusCode != 201) + { + Log.TraceEvent(TraceEventType.Error, 0, "Response Status Code {0}", response.StatusCode); + return false; + } + index += length; + bytesLeft -= length; + Debug.Assert(bytesLeft >= 0); + } } } - Log.TraceEvent(TraceEventType.Error, 0, "Response Status Code {0}", response.StatusCode); - return false; + return true; } static async ValueTask CopyStorageFileToLocalFile(StorageClient client, string storagePath, string localFilePath) diff --git a/samples/AzCopyCore/AzCopyCore/StorageRequests.cs b/samples/AzCopyCore/AzCopyCore/StorageRequests.cs index 12f0011aea1..2ea2294afde 100644 --- a/samples/AzCopyCore/AzCopyCore/StorageRequests.cs +++ b/samples/AzCopyCore/AzCopyCore/StorageRequests.cs @@ -102,23 +102,30 @@ public static ReadOnlyMemory AsString(Http.Method verb) public struct PutRangeRequest : IStorageRequest { - public Stream FileContent { get; set; } // TODO (pri 3): should there be a way to write from file handle or PipeReader? - public string FilePath { get; set; } + Stream _fileContent; // TODO (pri 3): should there be a way to write from file handle or PipeReader? + string _filePath; + long _offset; + int _length; // TODO (pri 3): I dont like how the client property is a public API public StorageClient Client { get; set; } - public PutRangeRequest(string filePath, Stream fileContent) + public PutRangeRequest(string filePath, Stream fileContent, long offset, int length) { - FilePath = filePath; - FileContent = fileContent; + if (offset < 0 || offset > fileContent.Length - length) throw new ArgumentOutOfRangeException(nameof(offset)); + if (length < 1) throw new ArgumentOutOfRangeException(nameof(length)); + + _filePath = filePath; + _fileContent = fileContent; Client = null; + _offset = offset; + _length = length; } - public long ContentLength => FileContent.Length; + public long ContentLength => _length; // TODO (pri 2): would be nice to elimnate these allocations - public string RequestPath => FilePath + "?comp=range"; - public string CanonicalizedResource => FilePath + "\ncomp:range"; + public string RequestPath => _filePath + "?comp=range"; + public string CanonicalizedResource => _filePath + "\ncomp:range"; public bool ConsumeBody => true; // TODO (pri 3): can this be an extension method? All implementations are the same. @@ -131,14 +138,21 @@ class Writer : StorageRequestWriter public override Http.Method Verb => Http.Method.Put; protected override async Task WriteBody(PipeWriter writer, PutRangeRequest arguments) - => await writer.WriteAsync(arguments.FileContent); + { + Stream stream = arguments._fileContent; + stream.Seek(arguments._offset, SeekOrigin.Begin); + await writer.WriteAsync(stream, arguments._length); + } protected override void WriteXmsHeaders(ref BufferWriter writer, ref PutRangeRequest arguments) { - long size = arguments.FileContent.Length; + long size = arguments._fileContent.Length; writer.WriteHeader("x-ms-date", Time, 'R'); // TODO (pri 3): this allocation should be eliminated - writer.WriteHeader("x-ms-range", $"bytes=0-{size - 1}"); + + long start = arguments._offset; + long end = start + arguments._length - 1; + writer.WriteHeader("x-ms-range", $"bytes={start}-{end}"); writer.WriteHeader("x-ms-version", "2017-04-17"); writer.WriteHeader("x-ms-write", "update"); }