From 34bbc4d987129c4f3e8a89f6c2c3270684fd260d Mon Sep 17 00:00:00 2001 From: Adam Date: Sun, 22 Aug 2021 12:23:51 -0400 Subject: [PATCH] feat: big fixes and improvements to kaitai #110 #119 --- src/hobbits-core/rangehighlight.cpp | 28 ++++-- src/hobbits-core/rangehighlight.h | 8 +- .../analyzers/KaitaiStruct/kaitaistruct.cpp | 42 +++++--- .../analyzers/KaitaiStruct/kaitaistruct.h | 9 +- .../analyzers/KaitaiStruct/scripts/runner.py | 97 +++++++------------ src/hobbits-widgets/highlightnavigator.cpp | 22 ++++- 6 files changed, 116 insertions(+), 90 deletions(-) diff --git a/src/hobbits-core/rangehighlight.cpp b/src/hobbits-core/rangehighlight.cpp index 6eb4c379..1d73994b 100644 --- a/src/hobbits-core/rangehighlight.cpp +++ b/src/hobbits-core/rangehighlight.cpp @@ -1,20 +1,22 @@ #include "rangehighlight.h" -RangeHighlight::RangeHighlight(QString category, QString label, Range range, quint32 color, QList children) : +RangeHighlight::RangeHighlight(QString category, QString label, Range range, quint32 color, QList children, QStringList tags) : m_category(category), m_label(label), m_range(range), m_color(color), - m_children(children) + m_children(children), + m_tags(tags) { } -RangeHighlight::RangeHighlight(QString category, QString label, QList children, quint32 color) : +RangeHighlight::RangeHighlight(QString category, QString label, QList children, quint32 color, QStringList tags) : m_category(category), m_label(label), m_color(color), - m_children(children) + m_children(children), + m_tags(tags) { std::sort(m_children.begin(), m_children.end()); if (m_children.isEmpty()) { @@ -23,9 +25,9 @@ RangeHighlight::RangeHighlight(QString category, QString label, QList RangeHighlight::allDescendants() const return all; } +QStringList RangeHighlight::tags() const +{ + return m_tags; +} + const QString VERSION_1 = "RangeHighlight v1"; const QString VERSION_2 = "RangeHighlight v2"; const QString VERSION_3 = "RangeHighlight v3"; +const QString VERSION_4 = "RHv4"; QDataStream& operator<<(QDataStream& stream, const RangeHighlight& highlight) { - stream << VERSION_3; + stream << VERSION_4; stream << highlight.category(); stream << highlight.label(); stream << highlight.range(); stream << highlight.color(); stream << highlight.children(); + stream << highlight.tags(); return stream; } @@ -86,12 +95,15 @@ QDataStream& operator>>(QDataStream& stream, RangeHighlight& highlight) { QString version; stream >> version; - if (version == VERSION_3) { + if (version == VERSION_3 || version == VERSION_4) { stream >> highlight.m_category; stream >> highlight.m_label; stream >> highlight.m_range; stream >> highlight.m_color; stream >> highlight.m_children; + if (version == VERSION_4) { + stream >> highlight.m_tags; + } return stream; } else { diff --git a/src/hobbits-core/rangehighlight.h b/src/hobbits-core/rangehighlight.h index cda10348..88f9b861 100644 --- a/src/hobbits-core/rangehighlight.h +++ b/src/hobbits-core/rangehighlight.h @@ -18,10 +18,10 @@ class HOBBITSCORESHARED_EXPORT RangeHighlight RangeHighlight(const RangeHighlight &) = default; RangeHighlight &operator=(const RangeHighlight &) = default; - RangeHighlight(QString category, QString label, Range range, quint32 color, QList children = {}); - RangeHighlight(QString category, QString label, QList children, quint32 color); + RangeHighlight(QString category, QString label, Range range, quint32 color, QList children = {}, QStringList tags = {}); + RangeHighlight(QString category, QString label, QList children, quint32 color, QStringList tags = {}); - static RangeHighlight simple(QString category, QString label, Range range, quint32 color); + static RangeHighlight simple(QString category, QString label, Range range, quint32 color, QStringList tags = {}); QString label() const; QString category() const; @@ -29,6 +29,7 @@ class HOBBITSCORESHARED_EXPORT RangeHighlight quint32 color() const; QList children() const; QList allDescendants() const; + QStringList tags() const; friend QDataStream& operator<<(QDataStream&, const RangeHighlight&); friend QDataStream& operator>>(QDataStream&, RangeHighlight&); @@ -39,6 +40,7 @@ class HOBBITSCORESHARED_EXPORT RangeHighlight Range m_range; quint32 m_color; QList m_children; + QStringList m_tags; }; bool HOBBITSCORESHARED_EXPORT operator<(const RangeHighlight &a, const RangeHighlight &b); diff --git a/src/hobbits-plugins/analyzers/KaitaiStruct/kaitaistruct.cpp b/src/hobbits-plugins/analyzers/KaitaiStruct/kaitaistruct.cpp index e346623b..71ed6ff5 100644 --- a/src/hobbits-plugins/analyzers/KaitaiStruct/kaitaistruct.cpp +++ b/src/hobbits-plugins/analyzers/KaitaiStruct/kaitaistruct.cpp @@ -259,7 +259,7 @@ QSharedPointer KaitaiStruct::analyzeBits( return AnalyzerResult::error("Output analysis file doesn't contain a 'sections' specification"); } QList highlights; - QMap>> labelMap; + QMap> fieldMap; QList topLevel; QJsonArray sections = outputObj.value("sections").toArray(); int sectionNum = 0; @@ -269,29 +269,39 @@ QSharedPointer KaitaiStruct::analyzeBits( } sectionNum++; QJsonObject s = section.toObject(); + QString label = QString("<%1>").arg(sectionNum); if (s.contains("label") && s.value("label").isString()) { label = s.value("label").toString(); } - if (!s.contains("start") || !s.contains("end") || !s.value("start").isDouble() || !s.value("end").isDouble()) { - labelMap.insert(label, {Range(), {}}); - } - else { + auto field = QSharedPointer(new KsField); + fieldMap.insert(label, field); + field->label = label; + + if (s.contains("start") && s.contains("end") && s.value("start").isDouble() && s.value("end").isDouble()) { Range range(qint64(s.value("start").toDouble())*8, qint64(s.value("end").toDouble())*8 - 1); - labelMap.insert(label, {range, {}}); + field->range = range; + } + + if (s.contains("value")) { + field->value = s.value("value").toVariant().toString(); + } + + if (s.contains("type")) { + field->type = s.value("type").toVariant().toString(); } if (!s.contains("parent") || s.value("parent").toString().isEmpty()) { topLevel.append(label); } - else if (labelMap.contains(s.value("parent").toString())) { - labelMap[s.value("parent").toString()].second.append(label); + else if (fieldMap.contains(s.value("parent").toString())) { + fieldMap[s.value("parent").toString()]->children.append(label); } } int colorIdx = 0; for (auto label : topLevel) { - highlights.append(makeHighlight(label, labelMap, colorIdx)); + highlights.append(makeHighlight(label, fieldMap, colorIdx)); } QSharedPointer bitInfo = BitInfo::copyFromContainer(container); @@ -308,7 +318,7 @@ QSharedPointer KaitaiStruct::analyzeBits( return AnalyzerResult::result(bitInfo, parameters); } -RangeHighlight KaitaiStruct::makeHighlight(QString label, const QMap>> &rangeData, int &colorIdx) +RangeHighlight KaitaiStruct::makeHighlight(QString label, const QMap> &fieldData, int &colorIdx) { QList colors = { QColor(100, 220, 100, 200), @@ -317,9 +327,9 @@ RangeHighlight KaitaiStruct::makeHighlight(QString label, const QMapchildren.isEmpty()) { + auto highlight = RangeHighlight(KAITAI_STRUCT_CATEGORY, label, field->range, colors.at(colorIdx).rgba(), {}, {field->type, field->value}); colorIdx = (colorIdx + 1) % colors.size(); return highlight; } @@ -327,11 +337,11 @@ RangeHighlight KaitaiStruct::makeHighlight(QString label, const QMap children; - for (auto child : pair.second) { - children.append(makeHighlight(child, rangeData, colorIdx)); + for (auto child : field->children) { + children.append(makeHighlight(child, fieldData, colorIdx)); } colorIdx = parentColorIndex; - auto highlight = RangeHighlight(KAITAI_STRUCT_CATEGORY, label, children, colors.at(colorIdx).rgba()); + auto highlight = RangeHighlight(KAITAI_STRUCT_CATEGORY, label, children, colors.at(colorIdx).rgba(), {field->type}); colorIdx = (colorIdx + 1) % colors.size(); return highlight; } diff --git a/src/hobbits-plugins/analyzers/KaitaiStruct/kaitaistruct.h b/src/hobbits-plugins/analyzers/KaitaiStruct/kaitaistruct.h index 77c39349..0ef8453d 100644 --- a/src/hobbits-plugins/analyzers/KaitaiStruct/kaitaistruct.h +++ b/src/hobbits-plugins/analyzers/KaitaiStruct/kaitaistruct.h @@ -28,7 +28,14 @@ class KaitaiStruct : public QObject, AnalyzerInterface private: - RangeHighlight makeHighlight(QString label, const QMap>> &rangeData, int &colorIdx); + typedef struct KsField { + QString label; + Range range; + QString value; + QString type; + QVector children; + } KsField; + RangeHighlight makeHighlight(QString label, const QMap> &fieldData, int &colorIdx); QSharedPointer m_delegate; diff --git a/src/hobbits-plugins/analyzers/KaitaiStruct/scripts/runner.py b/src/hobbits-plugins/analyzers/KaitaiStruct/scripts/runner.py index 07a09258..e246d387 100644 --- a/src/hobbits-plugins/analyzers/KaitaiStruct/scripts/runner.py +++ b/src/hobbits-plugins/analyzers/KaitaiStruct/scripts/runner.py @@ -6,55 +6,13 @@ import os import binascii import traceback +from enum import Enum from kaitaistruct import KaitaiStruct import pprint pp = pprint.PrettyPrinter(indent=2) -def dump_struct(s, sections, prefix="", parent_offset = 0, root_io=None, root_offset=0): - if isinstance(s, list): - print(f"\n[BEGIN List Under '{prefix}']") - for i, item in enumerate(s): - label = prefix + "[" + str(i) + "]" - print(label) - sections.append({ - "label": label, - "parent": prefix - }) - dump_struct(item, sections, label, parent_offset, root_io, root_offset) - print(f"[END OF List Under '{prefix}']\n") - elif isinstance(s, KaitaiStruct): - pp.pprint(vars(s)) - if hasattr(s, "_debug"): - - section_io = s._io - if root_io is None: - root_io = section_io - elif section_io is not root_io: - root_io = section_io - root_offset = root_offset + parent_offset - - print("\nITEMS {\n") - for name, descr in s._debug.items(): - prop = getattr(s, name) - label = prefix + "." + name if prefix else name - print(f"(\n name: {label},\n descr: {descr},\n prop: {prop},\n offset: {root_offset}\n)\n") - - sections.append({ - "start": descr["start"] + root_offset, - "end": descr["end"] + root_offset, - "label": label, - "parent": prefix - }) - - - parent_offset = descr["start"] + root_offset - - dump_struct(prop, sections, label, parent_offset, root_io, root_offset) - - print("}\n") - def process_value(value, section): @@ -65,6 +23,11 @@ def process_value(value, section): value = f"0x{binascii.hexlify(value).decode()}" section['value'] = value + elif isinstance(value, str): + if len(value) > 15: + value = f"{value[:12]}..." + section['value'] = value + elif isinstance(value, float): section['value'] = value @@ -74,17 +37,21 @@ def process_value(value, section): elif isinstance(value, bool): section['value'] = value + elif isinstance(value, Enum): + section['type'] = type(value).__name__ + section['value'] = value.name + else: return False return True -def parse_struct(struct, sections, prefix="", parent_offset = 0, root_io=None, root_offset=0): +def parse_struct(struct, sections, prefix="", parent_offset = 0, base_io=None, base_offset=0): if not isinstance(struct, KaitaiStruct): return - print(f"Parsing {struct} at '{prefix}'") + print(f"Parsing {type(struct).__name__} at '{prefix}'") # iterate through members in order to read lazy instances for attr in vars(type(struct)): @@ -101,13 +68,13 @@ def parse_struct(struct, sections, prefix="", parent_offset = 0, root_io=None, r return - section_io = struct._io - if root_io is None: - root_io = section_io - elif section_io is not root_io: - root_io = section_io - root_offset = root_offset + parent_offset + if base_io is None: + base_io = section_io + elif section_io is not base_io: + base_io = section_io + base_offset = base_offset + parent_offset + print(f"New base offset for {type(struct).__name__}: {base_offset}") for name, info in struct._debug.items(): try: @@ -117,7 +84,7 @@ def parse_struct(struct, sections, prefix="", parent_offset = 0, root_io=None, r continue label = prefix + "." + name if prefix else name - parent_offset = info["start"] + root_offset + parent_offset = info["start"] + base_offset #print(f"{name}:") #pp.pprint(info) @@ -127,35 +94,39 @@ def parse_struct(struct, sections, prefix="", parent_offset = 0, root_io=None, r continue section = { - "start": info["start"] + root_offset, - "end": info["end"] + root_offset, + "start": info["start"] + base_offset, + "end": info["end"] + base_offset, "label": label, "parent": prefix } if isinstance(value, KaitaiStruct): + section['type'] = type(value).__name__ sections.append(section) - parse_struct(value, sections, label, parent_offset, root_io, root_offset) + parse_struct(value, sections, label, parent_offset, base_io, base_offset) elif isinstance(value, list) and len(value) > 0: sections.append(section) for idx, value_item in enumerate(value): + if not struct._debug[name] or not struct._debug[name]['arr'] or idx >= len(struct._debug[name]['arr']): + continue + idx_label = f"{label}[{idx}]" idx_section = { "label": idx_label, "parent": label } + idx_offset = struct._debug[name]['arr'][idx]['start'] + base_offset + if isinstance(value_item, KaitaiStruct): sections.append(idx_section) - parse_struct(value_item, sections, idx_label, parent_offset, root_io, root_offset) + parse_struct(value_item, sections, idx_label, idx_offset, base_io, base_offset) else: - if not struct._debug[name] or not struct._debug[name]['arr'] or idx >= len(struct._debug[name]['arr']): - continue value_item_section = { - "start": struct._debug[name]['arr'][idx]['start'] + root_offset, - "end": struct._debug[name]['arr'][idx]['end'] + root_offset, + "start": idx_offset, + "end": struct._debug[name]['arr'][idx]['end'] + base_offset, "label": idx_section["label"], "parent": idx_section["parent"], } @@ -164,6 +135,10 @@ def parse_struct(struct, sections, prefix="", parent_offset = 0, root_io=None, r elif process_value(value, section): sections.append(section) + + else: + section["value"] = f"" + sections.append(section) @@ -195,7 +170,7 @@ def parse_data(input_filename, output_filename, action_progress): finally: target._io.close() - pp.pprint(sections) + #pp.pprint(sections) action_progress.set_progress_percent(80) diff --git a/src/hobbits-widgets/highlightnavigator.cpp b/src/hobbits-widgets/highlightnavigator.cpp index eea07a20..9b4d8d23 100644 --- a/src/hobbits-widgets/highlightnavigator.cpp +++ b/src/hobbits-widgets/highlightnavigator.cpp @@ -87,6 +87,10 @@ QTreeWidgetItem* HighlightNavigator::highlightToItem(const RangeHighlight &highl item->setData(0, Qt::UserRole + 1, ++count); + for (int i = 0; i < highlight.tags().size() && i < 5; i++) { + item->setData(i+1, Qt::DisplayRole, highlight.tags().at(i)); + } + for (auto child : highlight.children()) { item->addChild(highlightToItem(child, count)); } @@ -137,13 +141,29 @@ void HighlightNavigator::refresh() QList items; m_allHighlightCount = 0; + int maxColumns = 1; for (auto highlight: m_container->info()->highlights(m_category)) { - items.append(highlightToItem(highlight, m_allHighlightCount)); + auto item = highlightToItem(highlight, m_allHighlightCount); + maxColumns = qMax(item->columnCount(), maxColumns); + items.append(item); } + ui->tw_highlights->setColumnCount(maxColumns); ui->tw_highlights->addTopLevelItems(items); if (items.size() > 0) { ui->tw_highlights->setCurrentItem(items.first()); } + + if (maxColumns == 1) { + ui->tw_highlights->setHeaderHidden(true); + } + else { + // TODO: this is pretty garbage and should be better. + // not helped by the fact that this method seems to incorrectly + // get called 3 times per update + int w = qMax(250, ui->tw_highlights->width()/2); + ui->tw_highlights->setColumnWidth(0, w); + ui->tw_highlights->setHeaderHidden(false); + } } void HighlightNavigator::selectNext()