Fix Array of low cardinality types serialization/deserialization

clickhouse/CMakeLists.txt

@@ -7,6 +7,7 @@ SET ( clickhouse-cpp-lib-src
     base/wire_format.cpp
 
     columns/array.cpp
+    columns/column.cpp
     columns/date.cpp
     columns/decimal.cpp
     columns/enum.cpp

clickhouse/columns/array.cpp

@@ -54,22 +54,34 @@ void ColumnArray::Append(ColumnRef column) {
     }
 }
 
-bool ColumnArray::Load(InputStream* input, size_t rows) {
+bool ColumnArray::LoadPrefix(InputStream* input, size_t rows) {
     if (!rows) {
         return true;
     }
-    if (!offsets_->Load(input, rows)) {
+
+    return data_->LoadPrefix(input, rows);
+}
+
+bool ColumnArray::LoadBody(InputStream* input, size_t rows) {
+    if (!rows) {
+        return true;
+    }
+    if (!offsets_->LoadBody(input, rows)) {
         return false;
     }
-    if (!data_->Load(input, (*offsets_)[rows - 1])) {
+    if (!data_->LoadBody(input, (*offsets_)[rows - 1])) {
         return false;
     }
     return true;
 }
 
-void ColumnArray::Save(OutputStream* output) {
-    offsets_->Save(output);
-    data_->Save(output);
+void ColumnArray::SavePrefix(OutputStream* output) {
+    data_->SavePrefix(output);
+}
+
+void ColumnArray::SaveBody(OutputStream* output) {
+    offsets_->SaveBody(output);
+    data_->SaveBody(output);
 }
 
 void ColumnArray::Clear() {
@@ -92,6 +104,7 @@ void ColumnArray::OffsetsIncrease(size_t n) {
 }
 
 size_t ColumnArray::GetOffset(size_t n) const {
+
     return (n == 0) ? 0 : (*offsets_)[n - 1];
 }
 

clickhouse/columns/array.h

@@ -24,11 +24,17 @@ class ColumnArray : public Column {
     /// Appends content of given column to the end of current one.
     void Append(ColumnRef column) override;
 
+    /// Loads column prefix from input stream.
+    bool LoadPrefix(InputStream* input, size_t rows) override;
+
     /// Loads column data from input stream.
-    bool Load(InputStream* input, size_t rows) override;
+    bool LoadBody(InputStream* input, size_t rows) override;
+
+    /// Saves column prefix to output stream.
+    void SavePrefix(OutputStream* output) override;
 
     /// Saves column data to output stream.
-    void Save(OutputStream* output) override;
+    void SaveBody(OutputStream* output) override;
 
     /// Clear column data .
     void Clear() override;

clickhouse/columns/column.cpp

@@ -0,0 +1,24 @@
+#include "column.h"
+
+namespace clickhouse {
+
+bool Column::LoadPrefix(InputStream*, size_t) {
+    /// does nothing by default
+    return true;
+}
+
+bool Column::Load(InputStream* input, size_t rows) {
+    return LoadPrefix(input, rows) && LoadBody(input, rows);
+}
+
+void Column::SavePrefix(OutputStream*) {
+    /// does nothing by default
+}
+
+/// Saves column data to output stream.
+void Column::Save(OutputStream* output) {
+    SavePrefix(output);
+    SaveBody(output);
+}
+
+}

clickhouse/columns/column.h

@@ -42,11 +42,27 @@ class Column : public std::enable_shared_from_this<Column> {
     /// Appends content of given column to the end of current one.
     virtual void Append(ColumnRef column) = 0;
 
+    /// Template method to load column data from input stream. It'll call LoadPrefix and LoadBody.
+    /// Should be called only once from the client. Derived classes should not call it.
+    bool Load(InputStream* input, size_t rows);
+
+    /// Loads column prefix from input stream.
+    virtual bool LoadPrefix(InputStream* input, size_t rows);
+
     /// Loads column data from input stream.
-    virtual bool Load(InputStream* input, size_t rows) = 0;
+    virtual bool LoadBody(InputStream* input, size_t rows) = 0;
+
+    /// Saves column prefix to output stream. Column types with prefixes must implement it.
+    virtual void SavePrefix(OutputStream* output);
+
+    /// Saves column body to output stream.
+    virtual void SaveBody(OutputStream* output) = 0;
 
-    /// Saves column data to output stream.
-    virtual void Save(OutputStream* output) = 0;
+    /// Template method to save to output stream. It'll call SavePrefix and SaveBody respectively
+    /// Should be called only once from the client. Derived classes should not call it.
+    /// Save is split in Prefix and Body because some data types require prefixes and specific serialization order.
+    /// For instance, Array(LowCardinality(X)) requires LowCardinality.key_version bytes to come before Array.offsets
+    void Save(OutputStream* output);
 
     /// Clear column data .
     virtual void Clear() = 0;

clickhouse/columns/date.cpp

@@ -27,12 +27,12 @@ void ColumnDate::Append(ColumnRef column) {
     }
 }
 
-bool ColumnDate::Load(InputStream* input, size_t rows) {
-    return data_->Load(input, rows);
+bool ColumnDate::LoadBody(InputStream* input, size_t rows) {
+    return data_->LoadBody(input, rows);
 }
 
-void ColumnDate::Save(OutputStream* output) {
-    data_->Save(output);
+void ColumnDate::SaveBody(OutputStream* output) {
+    data_->SaveBody(output);
 }
 
 size_t ColumnDate::Size() const {
@@ -89,12 +89,12 @@ void ColumnDateTime::Append(ColumnRef column) {
     }
 }
 
-bool ColumnDateTime::Load(InputStream* input, size_t rows) {
-    return data_->Load(input, rows);
+bool ColumnDateTime::LoadBody(InputStream* input, size_t rows) {
+    return data_->LoadBody(input, rows);
 }
 
-void ColumnDateTime::Save(OutputStream* output) {
-    data_->Save(output);
+void ColumnDateTime::SaveBody(OutputStream* output) {
+    data_->SaveBody(output);
 }
 
 size_t ColumnDateTime::Size() const {
@@ -162,12 +162,12 @@ void ColumnDateTime64::Append(ColumnRef column) {
     }
 }
 
-bool ColumnDateTime64::Load(InputStream* input, size_t rows) {
-    return data_->Load(input, rows);
+bool ColumnDateTime64::LoadBody(InputStream* input, size_t rows) {
+    return data_->LoadBody(input, rows);
 }
 
-void ColumnDateTime64::Save(OutputStream* output) {
-    data_->Save(output);
+void ColumnDateTime64::SaveBody(OutputStream* output) {
+    data_->SaveBody(output);
 }
 
 void ColumnDateTime64::Clear() {

clickhouse/columns/date.h

@@ -26,10 +26,10 @@ class ColumnDate : public Column {
     void Append(ColumnRef column) override;
 
     /// Loads column data from input stream.
-    bool Load(InputStream* input, size_t rows) override;
+    bool LoadBody(InputStream* input, size_t rows) override;
 
     /// Saves column data to output stream.
-    void Save(OutputStream* output) override;
+    void SaveBody(OutputStream* output) override;
 
     /// Clear column data .
     void Clear() override;
@@ -70,13 +70,13 @@ class ColumnDateTime : public Column {
     void Append(ColumnRef column) override;
 
     /// Loads column data from input stream.
-    bool Load(InputStream* input, size_t rows) override;
+    bool LoadBody(InputStream* input, size_t rows) override;
 
     /// Clear column data .
     void Clear() override;
 
     /// Saves column data to output stream.
-    void Save(OutputStream* output) override;
+    void SaveBody(OutputStream* output) override;
 
     /// Returns count of rows in the column.
     size_t Size() const override;
@@ -118,13 +118,13 @@ class ColumnDateTime64 : public Column {
     void Append(ColumnRef column) override;
 
     /// Loads column data from input stream.
-    bool Load(InputStream* input, size_t rows) override;
+    bool LoadBody(InputStream* input, size_t rows) override;
 
     /// Clear column data .
     void Clear() override;
 
     /// Saves column data to output stream.
-    void Save(OutputStream* output) override;
+    void SaveBody(OutputStream* output) override;
 
     /// Returns count of rows in the column.
     size_t Size() const override;

clickhouse/columns/decimal.cpp

@@ -197,12 +197,12 @@ void ColumnDecimal::Append(ColumnRef column) {
     }
 }
 
-bool ColumnDecimal::Load(InputStream * input, size_t rows) {
-    return data_->Load(input, rows);
+bool ColumnDecimal::LoadBody(InputStream * input, size_t rows) {
+    return data_->LoadBody(input, rows);
 }
 
-void ColumnDecimal::Save(OutputStream* output) {
-    data_->Save(output);
+void ColumnDecimal::SaveBody(OutputStream* output) {
+    data_->SaveBody(output);
 }
 
 void ColumnDecimal::Clear() {

clickhouse/columns/decimal.h

@@ -21,8 +21,8 @@ class ColumnDecimal : public Column {
 
 public:
     void Append(ColumnRef column) override;
-    bool Load(InputStream* input, size_t rows) override;
-    void Save(OutputStream* output) override;
+    bool LoadBody(InputStream* input, size_t rows) override;
+    void SaveBody(OutputStream* output) override;
     void Clear() override;
     size_t Size() const override;
     ColumnRef Slice(size_t begin, size_t len) const override;

clickhouse/columns/enum.cpp

@@ -74,13 +74,13 @@ void ColumnEnum<T>::Append(ColumnRef column) {
 }
 
 template <typename T>
-bool ColumnEnum<T>::Load(InputStream* input, size_t rows) {
+bool ColumnEnum<T>::LoadBody(InputStream* input, size_t rows) {
     data_.resize(rows);
     return WireFormat::ReadBytes(*input, data_.data(), data_.size() * sizeof(T));
 }
 
 template <typename T>
-void ColumnEnum<T>::Save(OutputStream* output) {
+void ColumnEnum<T>::SaveBody(OutputStream* output) {
     WireFormat::WriteBytes(*output, data_.data(), data_.size() * sizeof(T));
 }
 

clickhouse/columns/enum.h

@@ -33,11 +33,11 @@ class ColumnEnum : public Column {
     void Append(ColumnRef column) override;
 
     /// Loads column data from input stream.
-    bool Load(InputStream* input, size_t rows) override;
+    bool LoadBody(InputStream* input, size_t rows) override;
 
     /// Saves column data to output stream.
-    void Save(OutputStream* output) override;
-    
+    void SaveBody(OutputStream* output) override;
+
     /// Clear column data .
     void Clear() override;
 

clickhouse/columns/ip4.cpp

@@ -71,12 +71,12 @@ void ColumnIPv4::Append(ColumnRef column) {
     }
 }
 
-bool ColumnIPv4::Load(InputStream * input, size_t rows) {
-    return data_->Load(input, rows);
+bool ColumnIPv4::LoadBody(InputStream * input, size_t rows) {
+    return data_->LoadBody(input, rows);
 }
 
-void ColumnIPv4::Save(OutputStream* output) {
-    data_->Save(output);
+void ColumnIPv4::SaveBody(OutputStream* output) {
+    data_->SaveBody(output);
 }
 
 size_t ColumnIPv4::Size() const {

clickhouse/columns/ip4.h

@@ -41,10 +41,10 @@ class ColumnIPv4 : public Column {
     void Append(ColumnRef column) override;
 
     /// Loads column data from input stream.
-    bool Load(InputStream* input, size_t rows) override;
+    bool LoadBody(InputStream* input, size_t rows) override;
 
     /// Saves column data to output stream.
-    void Save(OutputStream* output) override;
+    void SaveBody(OutputStream* output) override;
 
     /// Clear column data .
     void Clear() override;

clickhouse/columns/ip6.cpp

@@ -71,12 +71,12 @@ void ColumnIPv6::Append(ColumnRef column) {
     }
 }
 
-bool ColumnIPv6::Load(InputStream* input, size_t rows) {
-    return data_->Load(input, rows);
+bool ColumnIPv6::LoadBody(InputStream* input, size_t rows) {
+    return data_->LoadBody(input, rows);
 }
 
-void ColumnIPv6::Save(OutputStream* output) {
-    data_->Save(output);
+void ColumnIPv6::SaveBody(OutputStream* output) {
+    data_->SaveBody(output);
 }
 
 size_t ColumnIPv6::Size() const {

clickhouse/columns/ip6.h

@@ -39,10 +39,10 @@ class ColumnIPv6 : public Column {
     void Append(ColumnRef column) override;
 
     /// Loads column data from input stream.
-    bool Load(InputStream* input, size_t rows) override;
+    bool LoadBody(InputStream* input, size_t rows) override;
 
     /// Saves column data to output stream.
-    void Save(OutputStream* output) override;
+    void SaveBody(OutputStream* output) override;
 
     /// Clear column data .
     void Clear() override;