feat: add Bit Container import/export plugin

Also made some core updates to enable the de/serialization of bit containers
Mahlet-Inc · May 30, 2021 · 4907d2d · 4907d2d
1 parent 402fcf0
commit 4907d2d
Show file tree

Hide file tree

Showing 11 changed files with 353 additions and 5 deletions.
diff --git a/src/hobbits-core/bitarray.cpp b/src/hobbits-core/bitarray.cpp
@@ -4,6 +4,7 @@
 #include <QSharedPointer>
 #include <QString>
 #include <QDir>
+#include <QDataStream>
 #include <stdexcept>
 
 #ifdef Q_OS_UNIX
@@ -148,11 +149,19 @@ void BitArray::initFromIO(QIODevice *dataStream, qint64 sizeInBits)
     if (sizeInBits < 0) {
         sizeInBits = dataStream->bytesAvailable() * 8;
     }
+
+    QDataStream stream(dataStream);
+    initFromStream(stream, sizeInBits);
+}
+
+void BitArray::initFromStream(QDataStream &dataStream, qint64 sizeInBits)
+{
     m_size = sizeInBits;
     qint64 bytesToRead = this->sizeInBytes();
     char *byteBuffer = new char[CACHE_CHUNK_BYTE_SIZE];
     while (bytesToRead > 0) {
-        qint64 bytesRead = dataStream->read(byteBuffer, CACHE_CHUNK_BYTE_SIZE);
+        qint64 actualBytes = qMin(bytesToRead, qint64(CACHE_CHUNK_BYTE_SIZE));
+        qint64 bytesRead = dataStream.readRawData(byteBuffer, actualBytes);
         m_dataFile.write(byteBuffer, bytesRead);
         bytesToRead -= bytesRead;
 
@@ -629,13 +638,40 @@ QByteArray BitArray::readBytes(qint64 byteOffset, qint64 maxBytes) const
 }
 
 void BitArray::writeTo(QIODevice *outputStream) const
+{
+    QDataStream stream(outputStream);
+    writeToStream(stream);
+}
+
+BitArray* BitArray::deserialize(QDataStream &stream)
+{
+    qint64 sizeInBits;
+    stream >> sizeInBits;
+    if (sizeInBits < 0) {
+        stream.setStatus(QDataStream::Status::ReadCorruptData);
+        return nullptr;
+    }
+
+    auto bitArray = new BitArray();
+    bitArray->initFromStream(stream, sizeInBits);
+    return bitArray;
+}
+
+void BitArray::serialize(QDataStream &stream) const
+{
+    stream << m_size;
+    writeToStream(stream);
+}
+
+void BitArray::writeToStream(QDataStream &dataStream) const
 {
     QIODevice *reader = dataReader();
     char *byteBuffer = new char[CACHE_CHUNK_BYTE_SIZE];
     qint64 bytesToWrite = sizeInBytes();
     while (bytesToWrite > 0) {
-        qint64 bytesRead = reader->read(byteBuffer, CACHE_CHUNK_BYTE_SIZE);
-        outputStream->write(byteBuffer, bytesRead);
+        qint64 actualBytes = qMin(bytesToWrite, qint64(CACHE_CHUNK_BYTE_SIZE));
+        qint64 bytesRead = reader->read(byteBuffer, actualBytes);
+        dataStream.writeRawData(byteBuffer, bytesRead);
         bytesToWrite -= bytesRead;
 
         if (bytesToWrite > 0 && bytesRead < 1) {

diff --git a/src/hobbits-core/bitarray.h b/src/hobbits-core/bitarray.h
@@ -66,9 +66,14 @@ class HOBBITSCORESHARED_EXPORT BitArray
     QByteArray readBytes(qint64 byteOffset, qint64 maxBytes) const;
     void writeTo(QIODevice *outputStream) const;
 
+    static BitArray* deserialize(QDataStream &stream);
+    void serialize(QDataStream &stream) const;
+
     static QSharedPointer<BitArray> fromString(QString bitArraySpec, QStringList parseErrors = QStringList());
 
 private:
+    void writeToStream(QDataStream &dataStream) const; // private for use by serializer and writeTo
+
     class CacheLoadLocker {
     public:
         CacheLoadLocker(qint64 bitIndex, const BitArray* bitArray);
@@ -80,6 +85,7 @@ class HOBBITSCORESHARED_EXPORT BitArray
     QByteArray readBytesNoSync(qint64 byteOffset, qint64 maxBytes) const;
     QIODevice* dataReader() const;
     void initFromIO(QIODevice *dataStream, qint64 sizeInBits);
+    void initFromStream(QDataStream &dataStream, qint64 sizeInBits);
     void reinitializeCache();
     void deleteCache();
     void loadCacheAt(qint64 bitIndex) const;

diff --git a/src/hobbits-core/bitcontainer.cpp b/src/hobbits-core/bitcontainer.cpp
@@ -161,3 +161,38 @@ void BitContainer::addParent(QUuid parentId)
     QMutexLocker lock(&m_mutex);
     m_parents.append(parentId);
 }
+
+static const QString STREAM_HEADER_V1 = "HobbitsBitContainerStream_V1";
+
+QSharedPointer<BitContainer> BitContainer::deserialize(QDataStream &stream)
+{
+    int streamVersion;
+    QString streamHeader;
+
+    stream >> streamVersion;
+    stream.setVersion(streamVersion);
+
+    stream >> streamHeader;
+    if (streamHeader != STREAM_HEADER_V1) {
+        return QSharedPointer<BitContainer>();
+    }
+
+    auto container = QSharedPointer<BitContainer>(new BitContainer());
+
+    stream >> container->m_name;
+    stream >> container->m_nameWasSet;
+    container->m_bits = QSharedPointer<BitArray>(BitArray::deserialize(stream));
+    container->setInfo(BitInfo::deserialize(stream));
+
+    return container;
+}
+
+void BitContainer::serialize(QDataStream &stream) const
+{
+    stream << stream.version();
+    stream << STREAM_HEADER_V1;
+    stream << m_name;
+    stream << m_nameWasSet;
+    m_bits->serialize(stream);
+    m_info->serialize(stream);
+}
diff --git a/src/hobbits-core/bitcontainer.h b/src/hobbits-core/bitcontainer.h
@@ -64,6 +64,9 @@ class HOBBITSCORESHARED_EXPORT BitContainer : public QObject
     void addChild(QUuid childId);
     void addParent(QUuid parentId);
 
+    static QSharedPointer<BitContainer> deserialize(QDataStream &stream);
+    void serialize(QDataStream &stream) const;
+
 private:
     explicit BitContainer();
 

diff --git a/src/hobbits-core/bitinfo.cpp b/src/hobbits-core/bitinfo.cpp
@@ -155,3 +155,20 @@ qint64 BitInfo::frameOffsetContaining(qint64 value, Range indexBounds) const
 {
     return m_frames->indexOf(value, indexBounds);
 }
+
+QSharedPointer<BitInfo> BitInfo::deserialize(QDataStream &stream)
+{
+    auto info = new BitInfo();
+    info->m_frames = RangeSequence::deserialize(stream);
+    stream >> info->m_rangeHighlights;
+    stream >> info->m_metadata;
+
+    return QSharedPointer<BitInfo>(info);
+}
+
+void BitInfo::serialize(QDataStream &stream) const
+{
+    m_frames->serialize(stream);
+    stream << m_rangeHighlights;
+    stream << m_metadata;
+}
diff --git a/src/hobbits-core/bitinfo.h b/src/hobbits-core/bitinfo.h
@@ -43,8 +43,8 @@ class HOBBITSCORESHARED_EXPORT BitInfo : public QObject
 
     qint64 frameOffsetContaining(qint64 value, Range indexBounds = Range()) const;
 
-    friend QDataStream& operator<<(QDataStream&, const BitInfo&);
-    friend QDataStream& operator>>(QDataStream&, BitInfo&);
+    static QSharedPointer<BitInfo> deserialize(QDataStream &stream);
+    void serialize(QDataStream &stream) const;
 
 Q_SIGNALS:
     void changed();

diff --git a/src/hobbits-core/rangesequence.cpp b/src/hobbits-core/rangesequence.cpp
@@ -1,6 +1,7 @@
 #include "rangesequence.h"
 #include <QMutexLocker>
 #include <QSharedPointer>
+#include <QDataStream>
 #include <stdexcept>
 
 #define CACHE_CHUNK_64_SIZE (1000ll * 10ll)
@@ -299,3 +300,64 @@ qint64 RangeSequence::getValueCount() const
 {
     return m_valueCount;
 }
+
+QSharedPointer<RangeSequence> RangeSequence::deserialize(QDataStream &stream)
+{
+    qint64 valueCount;
+    qint64 constantSize;
+    stream >> valueCount;
+    stream >> constantSize;
+    if (constantSize > 0) {
+        return RangeSequence::fromConstantSize(constantSize, valueCount);
+    }
+
+    QSharedPointer<RangeSequence> sequence(new RangeSequence());
+
+    qint64 size;
+    qint64 maxSize;
+    int dataCacheBlockCount;
+    stream >> size;
+    stream >> maxSize;
+    stream >> dataCacheBlockCount;
+
+    sequence->m_valueCount = valueCount;
+    sequence->m_size = size;
+    sequence->m_maxSize = maxSize;
+    sequence->resizeCache(dataCacheBlockCount);
+    char* buffer = new char[CACHE_CHUNK_BYTE_SIZE];
+    for (int i = 0; i < dataCacheBlockCount; i++) {
+        int read = stream.readRawData(buffer, CACHE_CHUNK_BYTE_SIZE);
+        if (read < 1) {
+            stream.setStatus(QDataStream::Status::ReadCorruptData);
+            break;
+        }
+        sequence->m_dataFile.write(buffer, CACHE_CHUNK_BYTE_SIZE);
+    }
+    delete[] buffer;
+
+    return sequence;
+}
+
+void RangeSequence::serialize(QDataStream &stream) const
+{
+    stream << m_valueCount;
+    stream << m_constantSize;
+    if (m_constantSize > 0) {
+        return;
+    }
+
+    stream << m_size;
+    stream << m_maxSize;
+    stream << m_dataCacheBlockCount;
+    char* buffer = new char[CACHE_CHUNK_BYTE_SIZE];
+    syncCacheWithFile();
+    m_dataFile.seek(0);
+    while (m_dataFile.bytesAvailable() > 0) {
+        qint64 bytes = m_dataFile.read(buffer, CACHE_CHUNK_BYTE_SIZE);
+        if (bytes < 1) {
+            break;
+        }
+        stream.writeRawData(buffer, bytes);
+    }
+    delete[] buffer;
+}
diff --git a/src/hobbits-core/rangesequence.h b/src/hobbits-core/rangesequence.h
@@ -33,6 +33,9 @@ class HOBBITSCORESHARED_EXPORT RangeSequence
 
     qint64 getValueCount() const;
 
+    static QSharedPointer<RangeSequence> deserialize(QDataStream &stream);
+    void serialize(QDataStream &stream) const;
+
 private:
     void writeRange(qint64 i, qint64 end);
     Range readRange(qint64 i) const;

diff --git a/src/hobbits-plugins/importerexporters/BitContainerData/CMakeLists.txt b/src/hobbits-plugins/importerexporters/BitContainerData/CMakeLists.txt
@@ -0,0 +1 @@
+pluginInDir("${pluginType}" "BitContainerData" "${CMAKE_CURRENT_SOURCE_DIR}")
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		pluginInDir("${pluginType}" "BitContainerData" "${CMAKE_CURRENT_SOURCE_DIR}")