From a315b8a79fc034d9509b50ce3e0589291d30a0b3 Mon Sep 17 00:00:00 2001 From: maciejlach Date: Fri, 6 Jun 2014 14:34:52 +0200 Subject: [PATCH] Resolve #2: improve performance while retrieving large data samples --- CHANGELOG.txt | 9 ++++++++- qSharp/src/QBasicConnection.cs | 9 +++++++-- qSharp/src/QReader.cs | 8 ++++++-- qSharp/test/QReaderTest.cs | 6 +++--- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 1a29698..f77e45d 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,10 @@ +------------------------------------------------------------------------------ + qSharp 2.0.2 [2014.06.06] +------------------------------------------------------------------------------ + + - Improve performance while retrieving large data samples + - API: configurable maximum size of data chunk during data retrieval + ------------------------------------------------------------------------------ qSharp 2.0.1 [2014.05.26] ------------------------------------------------------------------------------ @@ -11,4 +18,4 @@ - Support for kdb+ protocol and types: v3.0, v2.6, v<=2.5 - Synchronous and asynchronous queries - Convenient asynchronous callbacks mechanism - - Logging of the raw protocol data in case of parsing failure. \ No newline at end of file + - Logging of the raw protocol data in case of parsing failure \ No newline at end of file diff --git a/qSharp/src/QBasicConnection.cs b/qSharp/src/QBasicConnection.cs index 442239a..3cd3e6c 100644 --- a/qSharp/src/QBasicConnection.cs +++ b/qSharp/src/QBasicConnection.cs @@ -31,11 +31,14 @@ namespace qSharp /// public class QBasicConnection : QConnection { + public const int DefaultMaxReadingChunk = 65536; + public string Host { get; private set; } public int Port { get; private set; } public string Username { get; private set; } public string Password { get; private set; } public System.Text.Encoding Encoding { get; private set; } + public int MaxReadingChunk { get; private set; } public int ProtocolVersion { get; protected set; } @@ -53,13 +56,15 @@ public class QBasicConnection : QConnection /// Username for remote authorization /// Password for remote authorization /// Encoding used for serialization/deserialization of string objects. Default: Encoding.ASCII - public QBasicConnection(String host = "localhost", int port = 0, string username = null, string password = null, Encoding encoding = null) + /// Maxium number of bytes read in a single chunk from stream + public QBasicConnection(String host = "localhost", int port = 0, string username = null, string password = null, Encoding encoding = null, int maxReadingChunk = DefaultMaxReadingChunk) { Encoding = encoding ?? Encoding.ASCII; Host = host; Port = port; Username = username; Password = password; + MaxReadingChunk = maxReadingChunk; } /// @@ -74,7 +79,7 @@ virtual public void Open() InitSocket(); Initialize(); - Reader = new QReader(stream, Encoding); + Reader = new QReader(stream, Encoding, MaxReadingChunk); Writer = new QWriter(stream, Encoding, ProtocolVersion); } else diff --git a/qSharp/src/QReader.cs b/qSharp/src/QReader.cs index 72200b0..14ed066 100644 --- a/qSharp/src/QReader.cs +++ b/qSharp/src/QReader.cs @@ -35,15 +35,19 @@ public sealed class QReader private byte[] rawData; private EndianBinaryReader reader; + private int maxReadingChunk; + /// /// Initializes a new QReader instance. /// /// Input stream containing serialized messages /// Encoding used for deserialization of string data - public QReader(Stream stream, Encoding encoding) + /// Maxium number of bytes read in a single chunk from stream + public QReader(Stream stream, Encoding encoding, int maxReadingChunk) { this.stream = stream; this.encoding = encoding; + this.maxReadingChunk = maxReadingChunk; } /// @@ -120,7 +124,7 @@ private byte[] ReadFully(int length) int read = 0; int chunk; - while ((chunk = stream.Read(buffer, read, buffer.Length - read)) > 0) + while ((chunk = stream.Read(buffer, read, Math.Min(maxReadingChunk, buffer.Length - read))) > 0) { read += chunk; diff --git a/qSharp/test/QReaderTest.cs b/qSharp/test/QReaderTest.cs index e523851..3f11881 100644 --- a/qSharp/test/QReaderTest.cs +++ b/qSharp/test/QReaderTest.cs @@ -52,7 +52,7 @@ public void TestCompressedObjectDeserializationQ2() { var stream = new MemoryStream(); var writer = new BinaryWriter(stream); - var reader = new QReader(stream, Encoding.ASCII); + var reader = new QReader(stream, Encoding.ASCII, QBasicConnection.DefaultMaxReadingChunk); byte[] binaryExpr = expressions.GetBinaryExpression("q2", expr); writer.Write((byte) 1); // little endian writer.Write((byte) 0); @@ -95,7 +95,7 @@ public void TestObjectDeserializationQ2() { var stream = new MemoryStream(); var writer = new BinaryWriter(stream); - var reader = new QReader(stream, Encoding.ASCII); + var reader = new QReader(stream, Encoding.ASCII, QBasicConnection.DefaultMaxReadingChunk); byte[] binaryExpr = expressions.GetBinaryExpression("q2", expr); writer.Write((byte) 1); // little endian writer.Write((byte) 0); @@ -142,7 +142,7 @@ public void TestObjectDeserializationQ3() { var stream = new MemoryStream(); var writer = new BinaryWriter(stream); - var reader = new QReader(stream, Encoding.ASCII); + var reader = new QReader(stream, Encoding.ASCII, QBasicConnection.DefaultMaxReadingChunk); byte[] binaryExpr = expressions.GetBinaryExpression("q3", expr); writer.Write((byte) 1); // little endian writer.Write((byte) 0);