Added the ability to upload files larger than 4MB (#2180)

* Bug fix in BufferWriter

* AzCopyCore: Added the ability to upload files larger than 4MB

* PR review feedback

samples/AzCopyCore/AzCopyCore/Helpers/PipelinesExtensions.cs

@@ -44,16 +44,21 @@ public static async Task WriteAsync(this Stream stream, PipeReader reader, ulong
         /// <summary>
         /// Copies bytes from Stream to PipeWriter 
         /// </summary>
-        public static async Task WriteAsync(this PipeWriter writer, Stream stream)
+        public static async Task WriteAsync(this PipeWriter writer, Stream stream, long bytesToWrite)
         {
             if (!stream.CanRead) throw new ArgumentException("Stream.CanRead returned false", nameof(stream));
-            while (true)
+            while (bytesToWrite > 0)
             {
                 Memory<byte> buffer = writer.GetMemory();
+                if (buffer.Length > bytesToWrite)
+                {
+                    buffer = buffer.Slice(0, (int)bytesToWrite);
+                }
                 if (buffer.Length == 0) throw new NotSupportedException("PipeWriter.GetMemory returned an empty buffer.");
                 int read = await stream.ReadAsync(buffer).ConfigureAwait(false);
                 if (read == 0) return;
                 writer.Advance(read);
+                bytesToWrite -= read;
                 await writer.FlushAsync();
             }
         }

samples/AzCopyCore/AzCopyCore/Program.cs

@@ -28,7 +28,7 @@ static void PrintUsage()
     static void Main(string[] args)
     {
         Log.Listeners.Add(new ConsoleTraceListener());
-        Log.Switch.Level = SourceLevels.Information;
+        Log.Switch.Level = SourceLevels.Error;
 
         var options = new CommandLine(args);
         ReadOnlySpan<char> source = options.GetSpan("/Source:");
@@ -197,14 +197,27 @@ static async ValueTask<bool> CopyLocalFileToStorageFile(StorageClient client, st
         {
             using (var bytes = new FileStream(localFilePath, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true))
             {
-                var putRequest = new PutRangeRequest(storagePath, bytes);
-                response = await client.SendRequest(putRequest).ConfigureAwait(false);
-                if (response.StatusCode == 201) return true;
+                long bytesLeft = bytes.Length;
+                long index = 0;
+                int length = 1024 * 1024 * 4; 
+                while (bytesLeft > 0)
+                {
+                    if (bytesLeft < length) length = (int)bytesLeft;
+                    var putRequest = new PutRangeRequest(storagePath, bytes, index, length);
+                    response = await client.SendRequest(putRequest).ConfigureAwait(false);
+                    if (response.StatusCode != 201)
+                    {
+                        Log.TraceEvent(TraceEventType.Error, 0, "Response Status Code {0}", response.StatusCode);
+                        return false;
+                    }
+                    index += length;
+                    bytesLeft -= length;
+                    Debug.Assert(bytesLeft >= 0);
+                }
             }
         }
 
-        Log.TraceEvent(TraceEventType.Error, 0, "Response Status Code {0}", response.StatusCode);
-        return false;
+        return true;
     }
 
     static async ValueTask<bool> CopyStorageFileToLocalFile(StorageClient client, string storagePath, string localFilePath)

samples/AzCopyCore/AzCopyCore/StorageRequests.cs

@@ -102,23 +102,30 @@ public static ReadOnlyMemory<byte> AsString(Http.Method verb)
 
     public struct PutRangeRequest : IStorageRequest
     {
-        public Stream FileContent { get; set; } // TODO (pri 3): should there be a way to write from file handle or PipeReader?
-        public string FilePath { get; set; }
+        Stream _fileContent; // TODO (pri 3): should there be a way to write from file handle or PipeReader?
+        string _filePath;
+        long _offset;
+        int _length;
 
         // TODO (pri 3): I dont like how the client property is a public API
         public StorageClient Client { get; set; }
 
-        public PutRangeRequest(string filePath, Stream fileContent)
+        public PutRangeRequest(string filePath, Stream fileContent, long offset, int length)
         {
-            FilePath = filePath;
-            FileContent = fileContent;
+            if (offset < 0 || offset > fileContent.Length - length) throw new ArgumentOutOfRangeException(nameof(offset));
+            if (length < 1) throw new ArgumentOutOfRangeException(nameof(length)); 
+
+            _filePath = filePath;
+            _fileContent = fileContent;
             Client = null;
+            _offset = offset;
+            _length = length;
         }
 
-        public long ContentLength => FileContent.Length;
+        public long ContentLength => _length;
         // TODO (pri 2): would be nice to elimnate these allocations
-        public string RequestPath => FilePath + "?comp=range";
-        public string CanonicalizedResource => FilePath + "\ncomp:range";
+        public string RequestPath => _filePath + "?comp=range";
+        public string CanonicalizedResource => _filePath + "\ncomp:range";
         public bool ConsumeBody => true;
 
         // TODO (pri 3): can this be an extension method? All implementations are the same.
@@ -131,14 +138,21 @@ class Writer : StorageRequestWriter<PutRangeRequest>
             public override Http.Method Verb => Http.Method.Put;
 
             protected override async Task WriteBody(PipeWriter writer, PutRangeRequest arguments)
-                => await writer.WriteAsync(arguments.FileContent);
+            {
+                Stream stream = arguments._fileContent;
+                stream.Seek(arguments._offset, SeekOrigin.Begin);
+                await writer.WriteAsync(stream, arguments._length);
+            }
 
             protected override void WriteXmsHeaders(ref BufferWriter writer, ref PutRangeRequest arguments)
             {
-                long size = arguments.FileContent.Length;
+                long size = arguments._fileContent.Length;
                 writer.WriteHeader("x-ms-date", Time, 'R');
                 // TODO (pri 3): this allocation should be eliminated
-                writer.WriteHeader("x-ms-range", $"bytes=0-{size - 1}");
+
+                long start = arguments._offset;
+                long end = start + arguments._length - 1;
+                writer.WriteHeader("x-ms-range", $"bytes={start}-{end}");
                 writer.WriteHeader("x-ms-version", "2017-04-17");
                 writer.WriteHeader("x-ms-write", "update");
             }