Feature/api v1

README.md

@@ -46,11 +46,6 @@ Once an analysis is complete, you may access RevEng.AI's BinNet embeddings for a
 reait -b /usr/bin/true -x > embeddings.json
 ```
 
-#### Extract embedding for symbol at vaddr 0x19F0
-```shell
-reait -b /usr/bin/true -x | jq ".[] | select(.vaddr==$((0x19F0))).embedding" > embedding.json
-```
-
 ### Search for similar symbols using an embedding
 To query our database of similar symbols based on an embedding, use `-n` to search using Approximate Nearest Neighbours. The `--nns` allows you to specify the number of results returned. A list of symbols with their names, distance (similarity), RevEng.AI collection set, source code filename, source code line number, and file creation timestamp is returned. 
 
@@ -151,7 +146,7 @@ Found /usr/bin/true:elf-x86_64
 ```shell
 apikey = "l1br3"
 host = "https://api.reveng.ai"
-model = "binnet-0.1"
+model = "binnet-0.3-x86"
 ```
 
 ## Contact

pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools >= 40.9.0", "tqdm", "argparse", "requests", "rich", "tomli", "pandas", "numpy", "scipy", "lief", "scikit-learn"]
+requires = ["setuptools >= 40.9.0", "argparse", "requests", "rich", "tomli", "pandas", "numpy", "scipy", "lief", "scikit-learn"]
 build-backend = "setuptools.build_meta"
 
 [project]
@@ -12,7 +12,6 @@ classifiers=[
   "Operating System :: OS Independent"
 ]
 dependencies = [
-    "tqdm",
     "requests",
     "rich",
     "argparse",

release.sh

@@ -17,4 +17,4 @@ echo "[?] Press enter to make the change. We will modify reait and pyproject.tom
 read -r line
 
 perl -i -pe "s/(?<=version.{0,10}\=.{0,10})[0-9]+\.[0-9]+\.[0-9]+/${VERSION}/" ./pyproject.toml
-perl -i -pe "s/(?<=version.{0,10}\=.{0,10})[0-9]+\.[0-9]+\.[0-9]+/${VERSION}/" ./src/reait/__init__.py
+perl -i -pe "s/(?<=version.{0,10}\=.{0,10})[0-9]+\.[0-9]+\.[0-9]+/${VERSION}/" ./src/reait/api.py

requirements.txt

@@ -0,0 +1,9 @@
+argparse
+requests
+rich
+tomli
+pandas
+numpy
+scipy
+scikit-learn
+lief

setup.py

@@ -3,7 +3,10 @@
 from setuptools import setup, find_packages
 
 
-with open("README.md") as fd:
+with open("requirements.txt") as fd:
+    required = fd.read().splitlines()
+
+with open("README.md", encoding="utf-8") as fd:
     long_description = fd.read()
 
 
@@ -14,6 +17,7 @@
     url="https://github.com/RevEng-AI/reait",
     author="James Patrick-Evans",
     author_email="[email protected]",
+    platforms="Cross Platform",
     packages=find_packages(where="src", exclude=["tests",]),
     package_dir={
         "": "src",
@@ -23,7 +27,5 @@
         "Programming Language :: Python :: 3",
         "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
     ],
-    install_requires=[
-        "tqdm", "argparse", "requests", "rich", "tomli", "pandas", "numpy", "scipy", "scikit-learn", "lief",
-    ],
+    install_requires=required,
 )

src/reait/__init__.py

@@ -1,6 +1,4 @@
 # -*- coding: utf-8 -*-
 from reait import api
-api.parse_config()
-
 
-__version__ = "1.0.0"
+api.parse_config()

src/reait/api.py

@@ -17,6 +17,7 @@
 from pandas import DataFrame
 from lief import parse, Binary, ELF, PE, MachO
 
+__version__ = "1.0.0"
 
 re_conf = {
     "apikey": "l1br3",
@@ -241,7 +242,8 @@ def RE_upload(fpath: str) -> Response:
                         '"message": "File already uploaded!",'
                         '"sha_256_hash": "{1}"{2}').format("{", bin_id, "}").encode()
     else:
-        res: Response = reveng_req(requests.post, "v1/upload", files={"file": open(fpath, "rb")})
+        with open(fpath, "rb") as fd:
+            res: Response = reveng_req(requests.post, "v1/upload", files={"file": fd})
 
         if res.ok:
             logger.info("Successfully uploaded binary to your account. %s - %s", fpath, bin_id)
@@ -555,39 +557,39 @@ def _binary_isa(binary: Binary, exec_type: str) -> str:
     """
     Get ISA format
     """
-    if exec_type == "elf":
+    if exec_type == "ELF":
         arch = binary.header.machine_type
 
         if arch == ELF.ARCH.i386:
             return "x86"
         elif arch == ELF.ARCH.x86_64:
             return "x86_64"
         elif arch == ELF.ARCH.ARM:
-            return "arm"
+            return "ARM32"
         elif arch == ELF.ARCH.AARCH64:
-            return "arm_64"
-    elif exec_type == "pe":
+            return "ARM64"
+    elif exec_type == "PE":
         machine_type = binary.header.machine
 
         if machine_type == PE.Header.MACHINE_TYPES.I386:
             return "x86"
         elif machine_type == PE.Header.MACHINE_TYPES.AMD64:
             return "x86_64"
         elif machine_type == PE.Header.MACHINE_TYPES.ARM:
-            return "arm"
+            return "ARM32"
         elif machine_type == PE.Header.MACHINE_TYPES.ARM64:
-            return "arm_64"
-    elif exec_type == "macho":
+            return "ARM64"
+    elif exec_type == "Mach-O":
         cpu_type = binary.header.cpu_type
 
         if cpu_type == MachO.CPU_TYPES.x86:
             return "x86"
         elif cpu_type == MachO.CPU_TYPES.x86_64:
             return "x86_64"
         elif cpu_type == MachO.CPU_TYPES.ARM:
-            return "arm"
+            return "ARM32"
         elif cpu_type == MachO.CPU_TYPES.ARM64:
-            return "arm_64"
+            return "ARM64"
 
     logger.error("Error, could not determine or unsupported ISA for binary format: %s.", exec_type)
     raise RuntimeError(f"Error, could not determine or unsupported ISA for binary format: {exec_type}.")
@@ -598,11 +600,11 @@ def _binary_format(binary: Binary) -> str:
     Get executable file format
     """
     if binary.format == Binary.FORMATS.PE:
-        return "pe"
+        return "PE"
     if binary.format == Binary.FORMATS.ELF:
-        return "elf"
+        return "ELF"
     if binary.format == Binary.FORMATS.MACHO:
-        return "macho"
+        return "Mach-O"
 
     logger.error("Error, could not determine or unsupported binary format: %s.", binary.format)
     raise RuntimeError(f"Error, could not determine or unsupported binary format: {binary.format}")
@@ -632,7 +634,7 @@ def parse_config() -> None:
     fpath = expanduser("~/.reait.toml")
 
     if isfile(fpath) and access(fpath, R_OK):
-        with open(fpath, "r") as fd:
+        with open(fpath) as fd:
             config = tomli.loads(fd.read())
 
             for key in ("apikey", "host", "model",):

src/reait/main.py

@@ -13,11 +13,11 @@
 import argparse
 import json
 from sys import exit, stdout, stderr
-from reait import api, __version__
 from scipy.spatial import distance
 from glob import iglob
 import numpy as np
 
+import api
 
 rerr = Console(file=stderr, width=180)
 rout = Console(file=stdout, width=180)
@@ -37,7 +37,7 @@ def version() -> int:
 ::  :::::::::::  :::
 ::  :::::  ::::  :::
 ::::::::    :::::::: [/bold blue]
-  [bold red]reait[/bold red] [bold bright_green]v{__version__}[/bold bright_green]
+  [bold red]reait[/bold red] [bold bright_green]v{api.__version__}[/bold bright_green]
 """)
     rout.print("[yellow]Config:[/yellow]")
     print_json(data=api.re_conf)
@@ -289,7 +289,7 @@ def main() -> int:
         except TypeError as e:
             rerr.print("[bold red][!] Error, please supply a valid binary file using '-b' flag.[/bold red]")
             rerr.print(f"[yellow] {e} [/yellow]")
-            return -1
+            return 0
         except Exception as e:
             rerr.print(f"[bold red][!] Error, binary exec type could not be verified:[/bold red] {args.binary}")
             rerr.print(f"[yellow] {e} [/yellow]")

tests/run_all_unittests.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+from os.path import isdir, dirname
+
+from unittest import TestLoader, TextTestRunner
+
+from utils import testlog
+
+
+def main() -> int:
+    if not isdir("tests"):
+        testlog.error("!! Please execute from the root directory of reait")
+        return 1
+    else:
+        tests = TestLoader().discover(dirname(__file__))
+        result = TextTestRunner(verbosity=2).run(tests)
+
+        return 0 if result.wasSuccessful() else 1
+
+
+if __name__ == "__main__":
+    exit(main())

tests/test_apis.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+from sys import path
+from pathlib import Path
+from unittest import main
+
+# Make it possible to run this file from the root dir of reait without installing reait
+path.insert(0, Path(__file__).parent.as_posix())
+
+from utils import BaseTestCase
+
+import reait.api as api
+
+
+class TestAPIs(BaseTestCase):
+    def test_0_conf(self):
+        self.assertGreaterEqual(len(api.re_conf), 3)
+        self.assertTrue(all(api.re_conf[key] for key in ("apikey", "host", "model",)))
+        self.assertNotEqual(api.re_conf["apikey"], "l1br3")
+        self.assertIn(self._platform, api.re_conf["model"])
+
+    def test_1_upload(self):
+        try:
+            response = api.RE_upload(self._fpath).json()
+
+            self.assertTrue(response["success"], "Upload file has failed")
+            self.assertEqual(response["sha_256_hash"], api.re_binary_id(self._fpath), "SHA-256 mismatch")
+        except Exception:
+            self.fail(f"Failed to upload {self._fpath}")
+
+    def test_2_analysis(self):
+        try:
+            response = api.RE_analyse(self._fpath, model_name="binnet-0.3-x86-linux", duplicate=True).json()
+
+            self.assertTrue(response["success"], "Analysis file has failed")
+            self.assertIsInstance(response["binary_id"], int)
+        except Exception:
+            self.fail(f"Failed to analyse {self._fpath}")
+
+    def test_3_analysis_failure(self):
+        try:
+            # Should raise a ReaitError because of duplicate analysis
+            api.RE_analyse(self._fpath, model_name="binnet-0.3-x86-linux")
+
+            self.fail(f"Duplicate analysis for {self._fpath}")
+        except Exception as e:
+            self.assertIsInstance(e, api.ReaitError)
+            self.assertIsNotNone(e.response)
+            self.assertEqual(e.response.status_code, 404)
+            self.assertFalse(e.response.json()["success"])
+
+    def test_4_logs(self):
+        try:
+            response = api.RE_logs(self._fpath).json()
+
+            self.assertTrue(response["success"], "Analysis file has failed")
+            self.assertIsNotNone(response["logs"], "Empty logs analysis")
+        except Exception:
+            self.fail("Failed to retrieve logs")
+
+    def test_5_delete(self):
+        try:
+            response = api.RE_delete(self._fpath).json()
+
+            self.assertTrue(response["success"], "Delete file has failed")
+        except Exception:
+            self.fail(f"Failed to delete {self._fpath}")
+
+
+if __name__ == "__main__":
+    main()

tests/test_reait.py

@@ -1,2 +1,87 @@
-def test_basic():
-    assert(True)
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+from subprocess import check_call
+from sys import executable
+from unittest import main
+
+from utils import BaseTestCase, testlog
+
+import reait.api as api
+
+
+def run_test_script(fpath: str, *args) -> int:
+    cmd = [executable, fpath] + list(args)
+
+    testlog.info("Running '%s'", " ".join(cmd))
+    return check_call(cmd, timeout=60)
+
+
+class TestReait(BaseTestCase):
+    def test_1_version(self):
+        try:
+            self.assertEqual(0,
+                             run_test_script("src/reait/main.py", "--version"))
+        except Exception as e:
+            testlog.error("Something went wrong when displaying version. %s", e)
+
+    def test_2_upload(self):
+        try:
+            self.assertEqual(0,
+                             run_test_script("src/reait/main.py",
+                                             "--binary", self._fpath, "--upload",
+                                             "--apikey", api.re_conf["apikey"]))
+        except Exception as e:
+            testlog.error("Something went wrong when upload binary for analysis. %s", e)
+
+    def test_3_analyse(self):
+        try:
+            self.assertEqual(0,
+                             run_test_script("src/reait/main.py",
+                                             "--binary", self._fpath, "--analyse", "--duplicate",
+                                             "--apikey", api.re_conf["apikey"],
+                                             "--model", api.re_conf["model"]))
+        except Exception as e:
+            testlog.error("Something went wrong when start analysis. %s", e)
+        finally:
+            self._cleanup_binaries(self._fpath)
+
+    def test_4_upload_analyse(self):
+        try:
+            self.assertEqual(0,
+                             run_test_script("src/reait/main.py",
+                                             "--binary", self._fpath, "-A", "--duplicate",
+                                             "--apikey", api.re_conf["apikey"],
+                                             "--model", api.re_conf["model"]))
+        except Exception as e:
+            testlog.error("Something went wrong when upload + start analysis. %s", e)
+
+    def test_5_logs(self):
+        try:
+            self.assertEqual(0,
+                             run_test_script("src/reait/main.py",
+                                             "--binary", self._fpath, "--logs",
+                                             "--apikey", api.re_conf["apikey"]))
+        except Exception as e:
+            testlog.error("Something went wrong when getting logs analysis. %s", e)
+
+    def test_6_status(self):
+        try:
+            self.assertEqual(0,
+                             run_test_script("src/reait/main.py",
+                                             "--binary", self._fpath, "--status",
+                                             "--apikey", api.re_conf["apikey"]))
+        except Exception as e:
+            testlog.error("Something went wrong when getting status. %s", e)
+
+    def test_7_delete(self):
+        try:
+            self.assertEqual(0,
+                             run_test_script("src/reait/main.py",
+                                             "--binary", self._fpath, "--delete",
+                                             "--apikey", api.re_conf["apikey"]))
+        except Exception as e:
+            testlog.error("Something went wrong when deleting analysis. %s", e)
+
+
+if __name__ == "__main__":
+    main()