From 5bc7dd2f84a1be966fe103f27cfdf02518c72949 Mon Sep 17 00:00:00 2001 From: Adam Nash Date: Sat, 5 Mar 2022 18:03:58 -0500 Subject: [PATCH] fix: rework some kaitai plugin stuff that was broken #132 --- .releaserc.json | 6 ++-- README.md | 6 +++- azure-pipelines.yml | 6 ++-- conanfile.py | 2 +- .../analyzers/KaitaiStruct/kaitaistruct.cpp | 18 ++++++++++-- .../analyzers/KaitaiStruct/scripts/runner.py | 28 +++++++++++++++---- 6 files changed, 50 insertions(+), 16 deletions(-) diff --git a/.releaserc.json b/.releaserc.json index 203bed82..79a283d6 100644 --- a/.releaserc.json +++ b/.releaserc.json @@ -50,9 +50,9 @@ "name": "hobbits-${nextRelease.version}-win2019.zip" }, { - "path": "hobbits/mac_1014_release/mac_1014_release.tar.gz", - "label": "Hobbits (64-bit Mac OSX 10.14)", - "name": "hobbits-${nextRelease.version}-mac1014.tgz" + "path": "hobbits/mac_1015_release/mac_1015_release.tar.gz", + "label": "Hobbits (64-bit Mac OSX 10.15)", + "name": "hobbits-${nextRelease.version}-mac1015.tgz" }, { "path": "hobbits/mac_11_release/mac_11_release.tar.gz", diff --git a/README.md b/README.md index 83ad299f..750843c8 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,11 @@ documentation and/or watch ``` conan remote add hobbitsconan https://helloadam.jfrog.io/artifactory/api/conan/conan ``` -3. Use conan to get/build dependencies, then build and package hobbits: +3. You might need libudev and patchelf for all of the conan steps to work. Ubuntu example: +``` +sudo apt-get install -y libudev-dev patchelf +``` +4. Use conan to get/build dependencies, then build and package hobbits: ``` mkdir build cd build diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7d5426ef..787aeaec 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -52,13 +52,13 @@ jobs: parameters: artifactName: win_2016_release -- job: Mac1014 +- job: Mac1015 pool: - vmImage: 'macOS-10.14' + vmImage: 'macOS-10.15' steps: - template: ci/mac.yml parameters: - artifactName: mac_1014_release + artifactName: mac_1015_release - job: Mac11 pool: diff --git a/conanfile.py b/conanfile.py index 21ffdd9c..f4b351fa 100644 --- a/conanfile.py +++ b/conanfile.py @@ -31,7 +31,7 @@ def set_version(self): requires = [ ("qt/5.15.2"), - ("hobbits-cpython/3.9.7"), + ("hobbits-cpython/3.9.10"), ("pffft/cci.20210511"), ("libusb/1.0.24") ] diff --git a/src/hobbits-plugins/analyzers/KaitaiStruct/kaitaistruct.cpp b/src/hobbits-plugins/analyzers/KaitaiStruct/kaitaistruct.cpp index 71ed6ff5..a1a3d2bc 100644 --- a/src/hobbits-plugins/analyzers/KaitaiStruct/kaitaistruct.cpp +++ b/src/hobbits-plugins/analyzers/KaitaiStruct/kaitaistruct.cpp @@ -91,6 +91,10 @@ QSharedPointer KaitaiStruct::analyzeBits( QString kscOutput = ""; + QString pyDepDir = dir.filePath("pydeps"); + + QString baseName = ""; + if (parameters.contains(PARAM_KSY) && !parameters.value(PARAM_KSY).toString().isEmpty()) { QString kscPath = SettingsManager::getPrivateSetting(KAITAI_PATH_KEY).toString(); @@ -108,12 +112,14 @@ QSharedPointer KaitaiStruct::analyzeBits( ksy.write(parameters.value(PARAM_KSY).toString().toLocal8Bit()); ksy.close(); + HobbitsPython::recursiveDirCopy(":/kaitaidata/ksy/common", dir.filePath("common")); + progress->setProgressPercent(20); #ifdef Q_OS_WIN QStringList kscAgs = {"/C", kscPath, "--debug", "--ksc-json-output", "-t", "python", ksy.fileName()}; #else - QStringList kscAgs = {"--debug", "--ksc-json-output", "-t", "python", ksy.fileName()}; + QStringList kscAgs = {"--debug", "--ksc-json-output", "-t", "python", "-I", ".", ksy.fileName()}; #endif QProcess kscProcess; QProcessEnvironment env = QProcessEnvironment::systemEnvironment(); @@ -156,6 +162,11 @@ QSharedPointer KaitaiStruct::analyzeBits( continue; } QJsonObject obj = json.value(key).toObject(); + + if (obj.contains("firstSpecName") && obj.value("firstSpecName").isString()) { + baseName = obj.value("firstSpecName").toString(); + } + if (!obj.contains("errors") || !obj.value("errors").isArray()) { continue; } @@ -199,7 +210,9 @@ QSharedPointer KaitaiStruct::analyzeBits( && !parameters.value(PARAM_PY).toString().isEmpty()) { QString precompiledFilePath = parameters.value(PARAM_PY).toString(); QFileInfo info(precompiledFilePath); + baseName = info.baseName(); QFile::copy(precompiledFilePath, dir.filePath(info.fileName())); + HobbitsPython::recursiveDirCopy(":/kaitaidata/ksy_py/common", dir.path()); } else { return AnalyzerResult::error("Invalid parameters - must have a precompiled py or a custom ksy spec"); @@ -207,14 +220,13 @@ QSharedPointer KaitaiStruct::analyzeBits( progress->setProgressPercent(40); - - QString pyDepDir = dir.filePath("pydeps"); HobbitsPython::recursiveDirCopy(":/kaitaistruct/scripts/dependencies", pyDepDir); auto pyRequest = PythonRequest::create(":/kaitaistruct/scripts/runner.py")->setFunctionName("parse_data"); pyRequest->addPathExtension(pyDepDir); pyRequest->addArg(PythonArg::qString(inputBitFile.fileName())); pyRequest->addArg(PythonArg::qString(outputRangeFile.fileName())); + pyRequest->addArg(PythonArg::qString(baseName)); auto watcher = HobbitsPython::getInstance().runProcessScript(pyRequest, true); watcher->watcher()->future().waitForFinished(); auto result = watcher->result(); diff --git a/src/hobbits-plugins/analyzers/KaitaiStruct/scripts/runner.py b/src/hobbits-plugins/analyzers/KaitaiStruct/scripts/runner.py index c1b642aa..9342356f 100644 --- a/src/hobbits-plugins/analyzers/KaitaiStruct/scripts/runner.py +++ b/src/hobbits-plugins/analyzers/KaitaiStruct/scripts/runner.py @@ -7,6 +7,7 @@ import binascii import traceback from enum import Enum +import shutil from kaitaistruct import KaitaiStruct @@ -146,15 +147,32 @@ def parse_struct(struct, sections, prefix="", parent_offset = 0, base_io=None, b -def parse_data(input_filename, output_filename, action_progress): +def parse_data(input_filename, output_filename, base_name, action_progress): # locate the compiled struct module scripts = glob.glob(os.path.join(os.path.dirname(input_filename), '*.py')) if len(scripts) < 1: raise FileNotFoundError('Could not find the expected python kaitai parser - did the kaitai struct compiler fail?') - module_file = os.path.basename(scripts[0]) - sys.path.append(os.path.dirname(scripts[0])) - package_name = os.path.splitext(module_file)[0] - class_name = "".join([s.capitalize() for s in package_name.split("_")]) + + if len(scripts) > 1: + thedir = os.path.dirname(input_filename) + allparsers = os.path.join(thedir, 'concatenated_parsers.py') + with open(allparsers,'wb') as concat: + for script in scripts: + with open(script,'rb') as fd: + shutil.copyfileobj(fd, concat) + + module_file = os.path.basename(allparsers) + sys.path.append(os.path.dirname(allparsers)) + package_name = os.path.splitext(module_file)[0] + if not base_name: + base_name = os.path.splitext(os.path.basename(scripts[0]))[0] + class_name = "".join([s.capitalize() for s in base_name.split("_")]) + + else: + module_file = os.path.basename(scripts[0]) + sys.path.append(os.path.dirname(scripts[0])) + package_name = os.path.splitext(module_file)[0] + class_name = "".join([s.capitalize() for s in package_name.split("_")]) try: del sys.modules[package_name] except KeyError: