feat(handler): add support for MSI files

Seems to work but had to DIY the parsing due to issues with olefile.

feat(handler): fix issues with MSI handler

Seems to work on both vanilla and padded MSI files. This could be
migrated to a fully Python-based implementation in the future using:

 * https://github.com/nightlark/pymsi
 * https://github.com/decalage2/olefile

As of v0.47, olefile does not handle padded MSIs properly so we
re-implement CFBF header parsing and compute the archive size ourselves.

Author: jcrussell

docs/handlers.md

@@ -6,6 +6,7 @@
     cab,
     cpio,
     dmg,
+    msi,
     par2,
     partclone,
     rar,
@@ -89,6 +90,7 @@
     arc.ARCHandler,
     arj.ARJHandler,
     cab.CABHandler,
+    msi.MsiHandler,
     tar.TarUstarHandler,
     tar.TarUnixHandler,
     cpio.PortableASCIIHandler,

python/unblob/handlers/__init__.py

@@ -0,0 +1,128 @@
+"""MSI Handler.
+
+Extracts MSIs using 7z with custom CFBF header parsing to compute the full
+archive size.
+"""
+
+import struct
+from typing import Optional
+
+from structlog import get_logger
+
+from unblob.extractors import Command
+
+from ...models import (
+    File,
+    HandlerDoc,
+    HandlerType,
+    HexString,
+    Reference,
+    StructHandler,
+    ValidChunk,
+)
+
+logger = get_logger()
+
+
+class MsiHandler(StructHandler):
+    NAME = "msi"
+
+    PATTERNS = [HexString("D0 CF 11 E0 A1 B1 1A E1")]
+    C_DEFINITIONS = r"""
+        typedef struct cfbf_header
+        {
+                                        // [offset from start (bytes), length (bytes)]
+            uint8 signature[8];         // [00H,08] {0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1,
+                                        // 0x1a, 0xe1} for current version
+            uint8 clsid[16];            // [08H,16] reserved must be zero (WriteClassStg/
+                                        // GetClassFile uses root directory class id)
+            uint16 minorVersion;        // [18H,02] minor version of the format: 33 is
+                                        // written by reference implementation
+            uint16 dllVersion;          // [1AH,02] major version of the dll/format: 3 for
+                                        // 512-byte sectors, 4 for 4 KB sectors
+            uint16 byteOrder;           // [1CH,02] 0xFFFE: indicates Intel byte-ordering
+            uint16 sectorShift;         // [1EH,02] size of sectors in power-of-two;
+                                        // typically 9 indicating 512-byte sectors
+            uint16 miniSectorShift;     // [20H,02] size of mini-sectors in power-of-two;
+                                        // typically 6 indicating 64-byte mini-sectors
+            uint16 reserved;            // [22H,02] reserved, must be zero
+            uint32 reserved1;           // [24H,04] reserved, must be zero
+            uint32 csectDir;            // [28H,04] must be zero for 512-byte sectors,
+                                        // number of SECTs in directory chain for 4 KB
+                                        // sectors
+            uint32 csectFat;            // [2CH,04] number of SECTs in the FAT chain
+            uint32 sectDirStart;        // [30H,04] first SECT in the directory chain
+            uint32 txSignature;         // [34H,04] signature used for transactions; must
+                                        // be zero. The reference implementation
+                                        // does not support transactions
+            uint32 miniSectorCutoff;    // [38H,04] maximum size for a mini stream;
+                                        // typically 4096 bytes
+            uint32 sectMiniFatStart;    // [3CH,04] first SECT in the MiniFAT chain
+            uint32 csectMiniFat;        // [40H,04] number of SECTs in the MiniFAT chain
+            uint32 sectDifStart;        // [44H,04] first SECT in the DIFAT chain
+            uint32 csectDif;            // [48H,04] number of SECTs in the DIFAT chain
+            uint32 sectFat[109];        // [4CH,436] the SECTs of first 109 FAT sectors
+         } cfbf_header_t;
+    """
+    HEADER_STRUCT = "cfbf_header_t"
+
+    EXTRACTOR = Command("7z", "x", "-p", "-y", "{inpath}", "-o{outdir}")
+
+    DOC = HandlerDoc(
+        name="MSI",
+        description="Microsoft Installer (MSI) files are used for the installation, maintenance, and removal of software.",
+        handler_type=HandlerType.ARCHIVE,
+        vendor="Microsoft",
+        references=[
+            Reference(
+                title="MSI File Format Documentation",
+                url="https://docs.microsoft.com/en-us/windows/win32/msi/overview-of-windows-installer",
+            ),
+            Reference(
+                title="Compound File Binary Format",
+                url="https://en.wikipedia.org/wiki/Compound_File_Binary_Format",
+            ),
+        ],
+        limitations=[],
+    )
+
+    def calculate_chunk(self, file: File, start_offset: int) -> Optional[ValidChunk]:
+        file.seek(start_offset)
+        header = self.parse_header(file)
+
+        # Size of MSI is based on the maximum used sector. Need to walk the
+        # DIFAT entries and find the maximum used sector to compute the size.
+        sector_size = 2**header.sectorShift
+        entries_per_sector = sector_size // 4
+
+        max_used_sector = 0
+
+        for sector_id, sect in enumerate(header.sectFat):
+            # skip empty
+            if sect == 0xFFFFFFFF:
+                continue
+
+            file.seek(start_offset + 512 + sect * sector_size)
+            raw_sector = file.read(sector_size)
+            entries = struct.unpack(f"<{entries_per_sector}I", raw_sector)
+
+            base_sector_id = sector_id * entries_per_sector
+            for entry_id in range(len(entries) - 1, -1, -1):
+                if entries[entry_id] == 0xFFFFFFFF:
+                    continue
+
+                # Found the highest id  on this page
+                max_id = base_sector_id + entry_id
+
+                max_used_sector = max(max_used_sector, max_id)
+
+                # Once we have found the first non-empty element, we are done
+                # with all IDs in this sector
+                break
+
+        total_size = 512 + ((max_used_sector + 1) * sector_size)
+
+        return ValidChunk(
+            start_offset=start_offset,
+            end_offset=start_offset + total_size,
+        )

python/unblob/handlers/archive/msi.py

@@ -54,7 +54,8 @@
 DEFAULT_PROCESS_NUM = multiprocessing.cpu_count()
 DEFAULT_SKIP_MAGIC = (
     "BFLT",
-    "Composite Document File V2 Document",
+    # Disabled for MSI files
+    # "Composite Document File V2 Document",
     "Erlang BEAM file",
     "GIF",
     "GNU message catalog",

python/unblob/processing.py

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dce9e456ace76b969fe0fe4d228bf096662c11d2376d99a9210f6364428a94c4
+size 1563648

tests/integration/archive/msi/__input__/7z2501.msi

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da8f4120ab4ffacb19067a26f6a8b2695e00ec19bcc48ff694349c62df1b330b
+size 1563680

tests/integration/archive/msi/__input__/7z2501.msi.padded

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa8e5036d973688f1e8622fbe9ab22e037346e0def0197bf5e7cdf37da4e223d
+size 3831808

tests/integration/archive/msi/__input__/putty-64bit-0.83-installer.msi

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12c87c542e1d4a39b47f176ffa5fd1691c98e5f9d502e6e46573962fb77c4510
+size 3831840

tests/integration/archive/msi/__input__/putty-64bit-0.83-installer.msi.padded

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb
+size 16

tests/integration/archive/msi/__output__/7z2501.msi.padded_extract/0-16.padding

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb
+size 16