Merge pull request #137 from Mahlet-Inc/develop

Develop

Author: hello-adam

.releaserc.json

@@ -50,9 +50,9 @@
                     "name": "hobbits-${nextRelease.version}-win2019.zip"
                 },
                 {
-                    "path": "hobbits/mac_1014_release/mac_1014_release.tar.gz",
-                    "label": "Hobbits (64-bit Mac OSX 10.14)",
-                    "name": "hobbits-${nextRelease.version}-mac1014.tgz"
+                    "path": "hobbits/mac_1015_release/mac_1015_release.tar.gz",
+                    "label": "Hobbits (64-bit Mac OSX 10.15)",
+                    "name": "hobbits-${nextRelease.version}-mac1015.tgz"
                 },
                 {
                     "path": "hobbits/mac_11_release/mac_11_release.tar.gz",

README.md

@@ -26,7 +26,11 @@ documentation and/or watch
 ```
 conan remote add hobbitsconan https://helloadam.jfrog.io/artifactory/api/conan/conan
 ```
-3. Use conan to get/build dependencies, then build and package hobbits:
+3. You might need libudev and patchelf for all of the conan steps to work. Ubuntu example:
+```
+sudo apt-get install -y libudev-dev patchelf
+```
+4. Use conan to get/build dependencies, then build and package hobbits:
 ```
 mkdir build
 cd build

azure-pipelines.yml

@@ -52,13 +52,13 @@ jobs:
       parameters:
         artifactName: win_2016_release
 
-- job: Mac1014
+- job: Mac1015
   pool:
-    vmImage: 'macOS-10.14'
+    vmImage: 'macOS-10.15'
   steps:
     - template: ci/mac.yml
       parameters:
-        artifactName: mac_1014_release
+        artifactName: mac_1015_release
 
 - job: Mac11
   pool:

ci/ubuntu-setup.yml

@@ -11,6 +11,9 @@ steps:
   - script: sudo apt-get -y install ninja-build
     displayName: 'Install Ninja'
 
+  - script: sudo apt-get -y install libudev-dev patchelf
+    displayName: 'Install libudev and patchelf'
+
   - task: UsePythonVersion@0 
     inputs:
       versionSpec: '3.7'

conanfile.py

@@ -31,7 +31,7 @@ def set_version(self):
 
     requires = [
         ("qt/5.15.2"),
-        ("hobbits-cpython/3.9.7"),
+        ("hobbits-cpython/3.9.10"),
         ("pffft/cci.20210511"),
         ("libusb/1.0.24")
     ]

src/hobbits-plugins/analyzers/KaitaiStruct/kaitaistruct.cpp

@@ -91,6 +91,10 @@ QSharedPointer<const AnalyzerResult> KaitaiStruct::analyzeBits(
 
     QString kscOutput = "";
 
+    QString pyDepDir = dir.filePath("pydeps");
+
+    QString baseName = "";
+
     if (parameters.contains(PARAM_KSY)
             && !parameters.value(PARAM_KSY).toString().isEmpty()) {
         QString kscPath = SettingsManager::getPrivateSetting(KAITAI_PATH_KEY).toString();
@@ -108,12 +112,14 @@ QSharedPointer<const AnalyzerResult> KaitaiStruct::analyzeBits(
         ksy.write(parameters.value(PARAM_KSY).toString().toLocal8Bit());
         ksy.close();
 
+        HobbitsPython::recursiveDirCopy(":/kaitaidata/ksy/common", dir.filePath("common"));
+
         progress->setProgressPercent(20);
 
     #ifdef Q_OS_WIN
         QStringList kscAgs = {"/C", kscPath, "--debug", "--ksc-json-output", "-t", "python", ksy.fileName()};
     #else
-        QStringList kscAgs = {"--debug", "--ksc-json-output", "-t", "python", ksy.fileName()};
+        QStringList kscAgs = {"--debug", "--ksc-json-output", "-t", "python", "-I", ".", ksy.fileName()};
     #endif
         QProcess kscProcess;
         QProcessEnvironment env = QProcessEnvironment::systemEnvironment();
@@ -156,6 +162,11 @@ QSharedPointer<const AnalyzerResult> KaitaiStruct::analyzeBits(
                 continue;
             }
             QJsonObject obj = json.value(key).toObject();
+
+            if (obj.contains("firstSpecName") && obj.value("firstSpecName").isString()) {
+                baseName = obj.value("firstSpecName").toString();
+            }
+
             if (!obj.contains("errors") || !obj.value("errors").isArray()) {
                 continue;
             }
@@ -199,22 +210,23 @@ QSharedPointer<const AnalyzerResult> KaitaiStruct::analyzeBits(
              && !parameters.value(PARAM_PY).toString().isEmpty()) {
         QString precompiledFilePath = parameters.value(PARAM_PY).toString();
         QFileInfo info(precompiledFilePath);
+        baseName = info.baseName();
         QFile::copy(precompiledFilePath, dir.filePath(info.fileName()));
+        HobbitsPython::recursiveDirCopy(":/kaitaidata/ksy_py/common", dir.path());
     }
     else {
         return AnalyzerResult::error("Invalid parameters - must have a precompiled py or a custom ksy spec");
     }
 
     progress->setProgressPercent(40);
 
-
-    QString pyDepDir = dir.filePath("pydeps");
     HobbitsPython::recursiveDirCopy(":/kaitaistruct/scripts/dependencies", pyDepDir);
 
     auto pyRequest = PythonRequest::create(":/kaitaistruct/scripts/runner.py")->setFunctionName("parse_data");
     pyRequest->addPathExtension(pyDepDir);
     pyRequest->addArg(PythonArg::qString(inputBitFile.fileName()));
     pyRequest->addArg(PythonArg::qString(outputRangeFile.fileName()));
+    pyRequest->addArg(PythonArg::qString(baseName));
     auto watcher = HobbitsPython::getInstance().runProcessScript(pyRequest, true);
     watcher->watcher()->future().waitForFinished();
     auto result = watcher->result();

src/hobbits-plugins/analyzers/KaitaiStruct/scripts/runner.py

@@ -7,6 +7,7 @@
 import binascii
 import traceback
 from enum import Enum
+import shutil
 
 from kaitaistruct import KaitaiStruct
 
@@ -146,15 +147,32 @@ def parse_struct(struct, sections, prefix="", parent_offset = 0, base_io=None, b
 
 
 
-def parse_data(input_filename, output_filename, action_progress):
+def parse_data(input_filename, output_filename, base_name, action_progress):
     # locate the compiled struct module
     scripts = glob.glob(os.path.join(os.path.dirname(input_filename), '*.py'))
     if len(scripts) < 1:
         raise FileNotFoundError('Could not find the expected python kaitai parser - did the kaitai struct compiler fail?')
-    module_file = os.path.basename(scripts[0])
-    sys.path.append(os.path.dirname(scripts[0]))
-    package_name = os.path.splitext(module_file)[0]
-    class_name = "".join([s.capitalize() for s in package_name.split("_")])
+
+    if len(scripts) > 1:
+        thedir = os.path.dirname(input_filename)
+        allparsers = os.path.join(thedir, 'concatenated_parsers.py')
+        with open(allparsers,'wb') as concat:
+            for script in scripts:
+                with open(script,'rb') as fd:
+                    shutil.copyfileobj(fd, concat)
+
+        module_file = os.path.basename(allparsers)
+        sys.path.append(os.path.dirname(allparsers))
+        package_name = os.path.splitext(module_file)[0]
+        if not base_name:
+            base_name = os.path.splitext(os.path.basename(scripts[0]))[0]
+        class_name = "".join([s.capitalize() for s in base_name.split("_")])
+
+    else:
+        module_file = os.path.basename(scripts[0])
+        sys.path.append(os.path.dirname(scripts[0]))
+        package_name = os.path.splitext(module_file)[0]
+        class_name = "".join([s.capitalize() for s in package_name.split("_")])
     try:
         del sys.modules[package_name]
     except KeyError: