Resolve #2: improve performance while retrieving large data samples

CHANGELOG.txt

@@ -1,3 +1,10 @@
+------------------------------------------------------------------------------
+  qSharp 2.0.2 [2014.06.06]
+------------------------------------------------------------------------------
+
+  - Improve performance while retrieving large data samples
+  - API: configurable maximum size of data chunk during data retrieval
+
 ------------------------------------------------------------------------------
   qSharp 2.0.1 [2014.05.26]
 ------------------------------------------------------------------------------
@@ -11,4 +18,4 @@
   - Support for kdb+ protocol and types: v3.0, v2.6, v<=2.5
   - Synchronous and asynchronous queries
   - Convenient asynchronous callbacks mechanism
-  - Logging of the raw protocol data in case of parsing failure. 
+  - Logging of the raw protocol data in case of parsing failure

qSharp/src/QBasicConnection.cs

@@ -31,11 +31,14 @@ namespace qSharp
     /// </summary>
     public class QBasicConnection : QConnection
     {
+        public const int DefaultMaxReadingChunk = 65536;
+
         public string Host { get; private set; }
         public int Port { get; private set; }
         public string Username { get; private set; }
         public string Password { get; private set; }
         public System.Text.Encoding Encoding { get; private set; }
+        public int MaxReadingChunk { get; private set; }
 
         public int ProtocolVersion { get; protected set; }
 
@@ -53,13 +56,15 @@ public class QBasicConnection : QConnection
         /// <param name="username">Username for remote authorization</param>
         /// <param name="password">Password for remote authorization</param>
         /// <param name="encoding">Encoding used for serialization/deserialization of string objects. Default: Encoding.ASCII</param>
-        public QBasicConnection(String host = "localhost", int port = 0, string username = null, string password = null, Encoding encoding = null)
+        /// <param name="maxReadingChunk">Maxium number of bytes read in a single chunk from stream</param>
+        public QBasicConnection(String host = "localhost", int port = 0, string username = null, string password = null, Encoding encoding = null, int maxReadingChunk = DefaultMaxReadingChunk)
         {
             Encoding = encoding ?? Encoding.ASCII;
             Host = host;
             Port = port;
             Username = username;
             Password = password;
+            MaxReadingChunk = maxReadingChunk;
         }
 
         /// <summary>
@@ -74,7 +79,7 @@ virtual public void Open()
                     InitSocket();
                     Initialize();
 
-                    Reader = new QReader(stream, Encoding);
+                    Reader = new QReader(stream, Encoding, MaxReadingChunk);
                     Writer = new QWriter(stream, Encoding, ProtocolVersion);
                 }
                 else

qSharp/src/QReader.cs

@@ -35,15 +35,19 @@ public sealed class QReader
         private byte[] rawData;
         private EndianBinaryReader reader;
 
+        private int maxReadingChunk;
+
         /// <summary>
         ///     Initializes a new QReader instance.
         /// </summary>
         /// <param name="stream">Input stream containing serialized messages</param>
         /// <param name="encoding">Encoding used for deserialization of string data</param>
-        public QReader(Stream stream, Encoding encoding)
+        /// <param name="maxReadingChunk">Maxium number of bytes read in a single chunk from stream</param>
+        public QReader(Stream stream, Encoding encoding, int maxReadingChunk)
         {
             this.stream = stream;
             this.encoding = encoding;
+            this.maxReadingChunk = maxReadingChunk;
         }
 
         /// <summary>
@@ -120,7 +124,7 @@ private byte[] ReadFully(int length)
             int read = 0;
             int chunk;
 
-            while ((chunk = stream.Read(buffer, read, buffer.Length - read)) > 0)
+            while ((chunk = stream.Read(buffer, read, Math.Min(maxReadingChunk, buffer.Length - read))) > 0)
             {
                 read += chunk;
 

qSharp/test/QReaderTest.cs

@@ -52,7 +52,7 @@ public void TestCompressedObjectDeserializationQ2()
             {
                 var stream = new MemoryStream();
                 var writer = new BinaryWriter(stream);
-                var reader = new QReader(stream, Encoding.ASCII);
+                var reader = new QReader(stream, Encoding.ASCII, QBasicConnection.DefaultMaxReadingChunk);
                 byte[] binaryExpr = expressions.GetBinaryExpression("q2", expr);
                 writer.Write((byte) 1); // little endian
                 writer.Write((byte) 0);
@@ -95,7 +95,7 @@ public void TestObjectDeserializationQ2()
             {
                 var stream = new MemoryStream();
                 var writer = new BinaryWriter(stream);
-                var reader = new QReader(stream, Encoding.ASCII);
+                var reader = new QReader(stream, Encoding.ASCII, QBasicConnection.DefaultMaxReadingChunk);
                 byte[] binaryExpr = expressions.GetBinaryExpression("q2", expr);
                 writer.Write((byte) 1); // little endian
                 writer.Write((byte) 0);
@@ -142,7 +142,7 @@ public void TestObjectDeserializationQ3()
             {
                 var stream = new MemoryStream();
                 var writer = new BinaryWriter(stream);
-                var reader = new QReader(stream, Encoding.ASCII);
+                var reader = new QReader(stream, Encoding.ASCII, QBasicConnection.DefaultMaxReadingChunk);
                 byte[] binaryExpr = expressions.GetBinaryExpression("q3", expr);
                 writer.Write((byte) 1); // little endian
                 writer.Write((byte) 0);