From 4907d2da31e817f17a0e670cdf04098925a20031 Mon Sep 17 00:00:00 2001 From: Adam Date: Sun, 30 May 2021 13:43:16 -0400 Subject: [PATCH] feat: add Bit Container import/export plugin Also made some core updates to enable the de/serialization of bit containers --- src/hobbits-core/bitarray.cpp | 42 ++++- src/hobbits-core/bitarray.h | 6 + src/hobbits-core/bitcontainer.cpp | 35 +++++ src/hobbits-core/bitcontainer.h | 3 + src/hobbits-core/bitinfo.cpp | 17 ++ src/hobbits-core/bitinfo.h | 4 +- src/hobbits-core/rangesequence.cpp | 62 ++++++++ src/hobbits-core/rangesequence.h | 3 + .../BitContainerData/CMakeLists.txt | 1 + .../BitContainerData/bitcontainerdata.cpp | 146 ++++++++++++++++++ .../BitContainerData/bitcontainerdata.h | 39 +++++ 11 files changed, 353 insertions(+), 5 deletions(-) create mode 100644 src/hobbits-plugins/importerexporters/BitContainerData/CMakeLists.txt create mode 100644 src/hobbits-plugins/importerexporters/BitContainerData/bitcontainerdata.cpp create mode 100644 src/hobbits-plugins/importerexporters/BitContainerData/bitcontainerdata.h diff --git a/src/hobbits-core/bitarray.cpp b/src/hobbits-core/bitarray.cpp index 39783cd3..9bee4290 100644 --- a/src/hobbits-core/bitarray.cpp +++ b/src/hobbits-core/bitarray.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #ifdef Q_OS_UNIX @@ -148,11 +149,19 @@ void BitArray::initFromIO(QIODevice *dataStream, qint64 sizeInBits) if (sizeInBits < 0) { sizeInBits = dataStream->bytesAvailable() * 8; } + + QDataStream stream(dataStream); + initFromStream(stream, sizeInBits); +} + +void BitArray::initFromStream(QDataStream &dataStream, qint64 sizeInBits) +{ m_size = sizeInBits; qint64 bytesToRead = this->sizeInBytes(); char *byteBuffer = new char[CACHE_CHUNK_BYTE_SIZE]; while (bytesToRead > 0) { - qint64 bytesRead = dataStream->read(byteBuffer, CACHE_CHUNK_BYTE_SIZE); + qint64 actualBytes = qMin(bytesToRead, qint64(CACHE_CHUNK_BYTE_SIZE)); + qint64 bytesRead = dataStream.readRawData(byteBuffer, actualBytes); m_dataFile.write(byteBuffer, bytesRead); bytesToRead -= bytesRead; @@ -629,13 +638,40 @@ QByteArray BitArray::readBytes(qint64 byteOffset, qint64 maxBytes) const } void BitArray::writeTo(QIODevice *outputStream) const +{ + QDataStream stream(outputStream); + writeToStream(stream); +} + +BitArray* BitArray::deserialize(QDataStream &stream) +{ + qint64 sizeInBits; + stream >> sizeInBits; + if (sizeInBits < 0) { + stream.setStatus(QDataStream::Status::ReadCorruptData); + return nullptr; + } + + auto bitArray = new BitArray(); + bitArray->initFromStream(stream, sizeInBits); + return bitArray; +} + +void BitArray::serialize(QDataStream &stream) const +{ + stream << m_size; + writeToStream(stream); +} + +void BitArray::writeToStream(QDataStream &dataStream) const { QIODevice *reader = dataReader(); char *byteBuffer = new char[CACHE_CHUNK_BYTE_SIZE]; qint64 bytesToWrite = sizeInBytes(); while (bytesToWrite > 0) { - qint64 bytesRead = reader->read(byteBuffer, CACHE_CHUNK_BYTE_SIZE); - outputStream->write(byteBuffer, bytesRead); + qint64 actualBytes = qMin(bytesToWrite, qint64(CACHE_CHUNK_BYTE_SIZE)); + qint64 bytesRead = reader->read(byteBuffer, actualBytes); + dataStream.writeRawData(byteBuffer, bytesRead); bytesToWrite -= bytesRead; if (bytesToWrite > 0 && bytesRead < 1) { diff --git a/src/hobbits-core/bitarray.h b/src/hobbits-core/bitarray.h index a5302c42..a58ea375 100644 --- a/src/hobbits-core/bitarray.h +++ b/src/hobbits-core/bitarray.h @@ -66,9 +66,14 @@ class HOBBITSCORESHARED_EXPORT BitArray QByteArray readBytes(qint64 byteOffset, qint64 maxBytes) const; void writeTo(QIODevice *outputStream) const; + static BitArray* deserialize(QDataStream &stream); + void serialize(QDataStream &stream) const; + static QSharedPointer fromString(QString bitArraySpec, QStringList parseErrors = QStringList()); private: + void writeToStream(QDataStream &dataStream) const; // private for use by serializer and writeTo + class CacheLoadLocker { public: CacheLoadLocker(qint64 bitIndex, const BitArray* bitArray); @@ -80,6 +85,7 @@ class HOBBITSCORESHARED_EXPORT BitArray QByteArray readBytesNoSync(qint64 byteOffset, qint64 maxBytes) const; QIODevice* dataReader() const; void initFromIO(QIODevice *dataStream, qint64 sizeInBits); + void initFromStream(QDataStream &dataStream, qint64 sizeInBits); void reinitializeCache(); void deleteCache(); void loadCacheAt(qint64 bitIndex) const; diff --git a/src/hobbits-core/bitcontainer.cpp b/src/hobbits-core/bitcontainer.cpp index c01a8e48..161ee6b1 100644 --- a/src/hobbits-core/bitcontainer.cpp +++ b/src/hobbits-core/bitcontainer.cpp @@ -161,3 +161,38 @@ void BitContainer::addParent(QUuid parentId) QMutexLocker lock(&m_mutex); m_parents.append(parentId); } + +static const QString STREAM_HEADER_V1 = "HobbitsBitContainerStream_V1"; + +QSharedPointer BitContainer::deserialize(QDataStream &stream) +{ + int streamVersion; + QString streamHeader; + + stream >> streamVersion; + stream.setVersion(streamVersion); + + stream >> streamHeader; + if (streamHeader != STREAM_HEADER_V1) { + return QSharedPointer(); + } + + auto container = QSharedPointer(new BitContainer()); + + stream >> container->m_name; + stream >> container->m_nameWasSet; + container->m_bits = QSharedPointer(BitArray::deserialize(stream)); + container->setInfo(BitInfo::deserialize(stream)); + + return container; +} + +void BitContainer::serialize(QDataStream &stream) const +{ + stream << stream.version(); + stream << STREAM_HEADER_V1; + stream << m_name; + stream << m_nameWasSet; + m_bits->serialize(stream); + m_info->serialize(stream); +} diff --git a/src/hobbits-core/bitcontainer.h b/src/hobbits-core/bitcontainer.h index 55405c74..4ad798bd 100644 --- a/src/hobbits-core/bitcontainer.h +++ b/src/hobbits-core/bitcontainer.h @@ -64,6 +64,9 @@ class HOBBITSCORESHARED_EXPORT BitContainer : public QObject void addChild(QUuid childId); void addParent(QUuid parentId); + static QSharedPointer deserialize(QDataStream &stream); + void serialize(QDataStream &stream) const; + private: explicit BitContainer(); diff --git a/src/hobbits-core/bitinfo.cpp b/src/hobbits-core/bitinfo.cpp index 64df654c..a02b197e 100644 --- a/src/hobbits-core/bitinfo.cpp +++ b/src/hobbits-core/bitinfo.cpp @@ -155,3 +155,20 @@ qint64 BitInfo::frameOffsetContaining(qint64 value, Range indexBounds) const { return m_frames->indexOf(value, indexBounds); } + +QSharedPointer BitInfo::deserialize(QDataStream &stream) +{ + auto info = new BitInfo(); + info->m_frames = RangeSequence::deserialize(stream); + stream >> info->m_rangeHighlights; + stream >> info->m_metadata; + + return QSharedPointer(info); +} + +void BitInfo::serialize(QDataStream &stream) const +{ + m_frames->serialize(stream); + stream << m_rangeHighlights; + stream << m_metadata; +} diff --git a/src/hobbits-core/bitinfo.h b/src/hobbits-core/bitinfo.h index 0306726c..d3535872 100644 --- a/src/hobbits-core/bitinfo.h +++ b/src/hobbits-core/bitinfo.h @@ -43,8 +43,8 @@ class HOBBITSCORESHARED_EXPORT BitInfo : public QObject qint64 frameOffsetContaining(qint64 value, Range indexBounds = Range()) const; - friend QDataStream& operator<<(QDataStream&, const BitInfo&); - friend QDataStream& operator>>(QDataStream&, BitInfo&); + static QSharedPointer deserialize(QDataStream &stream); + void serialize(QDataStream &stream) const; Q_SIGNALS: void changed(); diff --git a/src/hobbits-core/rangesequence.cpp b/src/hobbits-core/rangesequence.cpp index b3ce4797..ae53b74b 100644 --- a/src/hobbits-core/rangesequence.cpp +++ b/src/hobbits-core/rangesequence.cpp @@ -1,6 +1,7 @@ #include "rangesequence.h" #include #include +#include #include #define CACHE_CHUNK_64_SIZE (1000ll * 10ll) @@ -299,3 +300,64 @@ qint64 RangeSequence::getValueCount() const { return m_valueCount; } + +QSharedPointer RangeSequence::deserialize(QDataStream &stream) +{ + qint64 valueCount; + qint64 constantSize; + stream >> valueCount; + stream >> constantSize; + if (constantSize > 0) { + return RangeSequence::fromConstantSize(constantSize, valueCount); + } + + QSharedPointer sequence(new RangeSequence()); + + qint64 size; + qint64 maxSize; + int dataCacheBlockCount; + stream >> size; + stream >> maxSize; + stream >> dataCacheBlockCount; + + sequence->m_valueCount = valueCount; + sequence->m_size = size; + sequence->m_maxSize = maxSize; + sequence->resizeCache(dataCacheBlockCount); + char* buffer = new char[CACHE_CHUNK_BYTE_SIZE]; + for (int i = 0; i < dataCacheBlockCount; i++) { + int read = stream.readRawData(buffer, CACHE_CHUNK_BYTE_SIZE); + if (read < 1) { + stream.setStatus(QDataStream::Status::ReadCorruptData); + break; + } + sequence->m_dataFile.write(buffer, CACHE_CHUNK_BYTE_SIZE); + } + delete[] buffer; + + return sequence; +} + +void RangeSequence::serialize(QDataStream &stream) const +{ + stream << m_valueCount; + stream << m_constantSize; + if (m_constantSize > 0) { + return; + } + + stream << m_size; + stream << m_maxSize; + stream << m_dataCacheBlockCount; + char* buffer = new char[CACHE_CHUNK_BYTE_SIZE]; + syncCacheWithFile(); + m_dataFile.seek(0); + while (m_dataFile.bytesAvailable() > 0) { + qint64 bytes = m_dataFile.read(buffer, CACHE_CHUNK_BYTE_SIZE); + if (bytes < 1) { + break; + } + stream.writeRawData(buffer, bytes); + } + delete[] buffer; +} diff --git a/src/hobbits-core/rangesequence.h b/src/hobbits-core/rangesequence.h index 0e205d45..0623d403 100644 --- a/src/hobbits-core/rangesequence.h +++ b/src/hobbits-core/rangesequence.h @@ -33,6 +33,9 @@ class HOBBITSCORESHARED_EXPORT RangeSequence qint64 getValueCount() const; + static QSharedPointer deserialize(QDataStream &stream); + void serialize(QDataStream &stream) const; + private: void writeRange(qint64 i, qint64 end); Range readRange(qint64 i) const; diff --git a/src/hobbits-plugins/importerexporters/BitContainerData/CMakeLists.txt b/src/hobbits-plugins/importerexporters/BitContainerData/CMakeLists.txt new file mode 100644 index 00000000..def650d5 --- /dev/null +++ b/src/hobbits-plugins/importerexporters/BitContainerData/CMakeLists.txt @@ -0,0 +1 @@ +pluginInDir("${pluginType}" "BitContainerData" "${CMAKE_CURRENT_SOURCE_DIR}") diff --git a/src/hobbits-plugins/importerexporters/BitContainerData/bitcontainerdata.cpp b/src/hobbits-plugins/importerexporters/BitContainerData/bitcontainerdata.cpp new file mode 100644 index 00000000..bb76fec7 --- /dev/null +++ b/src/hobbits-plugins/importerexporters/BitContainerData/bitcontainerdata.cpp @@ -0,0 +1,146 @@ +#include "bitcontainerdata.h" +#include "parametereditorfileselect.h" +#include "settingsmanager.h" + +BitContainerData::BitContainerData() +{ + QList infos = { + {"filename", ParameterDelegate::ParameterType::String} + }; + + m_importDelegate = ParameterDelegate::create( + infos, + [this](const Parameters ¶meters) { + if (parameters.contains("filename")) { + return QString("Import container from %1").arg(parameters.value("filename").toString()); + } + else { + return QString(); + } + }, + [](QSharedPointer delegate, QSize size) { + Q_UNUSED(size) + Q_UNUSED(delegate) + return new ParameterEditorFileSelect(QFileDialog::AcceptOpen); + }); + + m_exportDelegate = ParameterDelegate::create( + infos, + [this](const Parameters ¶meters) { + if (parameters.contains("filename")) { + return QString("Export container to %1").arg(parameters.value("filename").toString()); + } + else { + return QString(); + } + }, + [](QSharedPointer delegate, QSize size) { + Q_UNUSED(size) + Q_UNUSED(delegate) + return new ParameterEditorFileSelect(QFileDialog::AcceptSave); + }); +} + +ImporterExporterInterface* BitContainerData::createDefaultImporterExporter() +{ + return new BitContainerData(); +} + +QString BitContainerData::name() +{ + return "Hobbits Bit Container"; +} + +QString BitContainerData::description() +{ + return "Imports and exports Hobbits bit containers with their metadata"; +} + +QStringList BitContainerData::tags() +{ + return {"Generic"}; +} + +bool BitContainerData::canExport() +{ + return true; +} + +bool BitContainerData::canImport() +{ + return true; +} + +QSharedPointer BitContainerData::importParameterDelegate() +{ + return m_importDelegate; +} + +QSharedPointer BitContainerData::exportParameterDelegate() +{ + return m_exportDelegate; +} + +static const QString LAST_BIT_CONTAINER_IMPORT_EXPORT = "last_bit_container_import_export"; + +QSharedPointer BitContainerData::importBits(const Parameters ¶meters, + QSharedPointer progress) +{ + Q_UNUSED(progress) + QStringList invalidations = m_importDelegate->validate(parameters); + if (!invalidations.isEmpty()) { + return ImportResult::error(QString("Invalid parameters passed to %1:\n%2").arg(name()).arg(invalidations.join("\n"))); + } + QString fileName = parameters.value("filename").toString(); + if (fileName.isEmpty()) { + return ImportResult::error("No file selected for import"); + } + QFile file(fileName); + if (!file.open(QIODevice::ReadOnly)) { + return ImportResult::error(QString("Failed to open file for import: '%1'").arg(fileName)); + } + SettingsManager::setPrivateSetting( + LAST_BIT_CONTAINER_IMPORT_EXPORT, + QFileInfo(file).dir().path()); + + QDataStream stream(&file); + QSharedPointer container = BitContainer::deserialize(stream); + if (container.isNull()) { + return ImportResult::error(QString("Failed to load Bit Container from %1\n\nIs it a valid Bit Container file?").arg(fileName)); + } + container->setName(QFileInfo(file).fileName()); + + return ImportResult::result(container, parameters); +} + +QSharedPointer BitContainerData::exportBits(QSharedPointer container, + const Parameters ¶meters, + QSharedPointer progress) +{ + QStringList invalidations = m_exportDelegate->validate(parameters); + if (!invalidations.isEmpty()) { + return ExportResult::error(QString("Invalid parameters passed to %1:\n%2").arg(name()).arg(invalidations.join("\n"))); + } + progress->setProgressPercent(10); + + QString fileName = parameters.value("filename").toString(); + if (fileName.isEmpty()) { + return ExportResult::error("No file selected for export"); + } + + QFile file(fileName); + if (!file.open(QIODevice::Truncate | QIODevice::WriteOnly)) { + return ExportResult::error(QString("Failed to open export bit file: '%1'").arg(fileName)); + } + SettingsManager::setPrivateSetting( + SettingsManager::LAST_IMPORT_EXPORT_PATH_KEY, + QFileInfo(file).dir().path()); + + QDataStream stream(&file); + container->serialize(stream); + file.close(); + + progress->setProgressPercent(90); + + return ExportResult::result(parameters); +} diff --git a/src/hobbits-plugins/importerexporters/BitContainerData/bitcontainerdata.h b/src/hobbits-plugins/importerexporters/BitContainerData/bitcontainerdata.h new file mode 100644 index 00000000..95d2d8a7 --- /dev/null +++ b/src/hobbits-plugins/importerexporters/BitContainerData/bitcontainerdata.h @@ -0,0 +1,39 @@ +#ifndef BITCONTAINERDATA_H +#define BITCONTAINERDATA_H + +#include "importexportinterface.h" +#include "parameterdelegate.h" + +class BitContainerData : public QObject, ImporterExporterInterface +{ + Q_OBJECT + Q_PLUGIN_METADATA(IID "hobbits.ImporterExporterInterface.BitContainerData") + Q_INTERFACES(ImporterExporterInterface) + +public: + BitContainerData(); + + ImporterExporterInterface* createDefaultImporterExporter() override; + + QString name() override; + QString description() override; + QStringList tags() override; + + bool canExport() override; + bool canImport() override; + + virtual QSharedPointer importParameterDelegate() override; + virtual QSharedPointer exportParameterDelegate() override; + + QSharedPointer importBits(const Parameters ¶meters, + QSharedPointer progress) override; + QSharedPointer exportBits(QSharedPointer container, + const Parameters ¶meters, + QSharedPointer progress) override; + +private: + QSharedPointer m_importDelegate; + QSharedPointer m_exportDelegate; +}; + +#endif // BITCONTAINERDATA_H