Add ARM64 symbol cache

.github/workflows/deploy.yml

@@ -90,6 +90,7 @@ jobs:
           gh extension install actions/gh-actions-cache
           gh actions-cache delete cache-binaries-and-symbols-key-v1 --confirm -R ${{ github.repository }}
       - name: Save cache - binaries and symbols
+        if: false  # <- Temp, TODO: remove
         uses: actions/cache/save@v4
         with:
           path: binaries

scripts/01_extract_mod_symbols.py

@@ -2,12 +2,15 @@
 import re
 import time
 from argparse import ArgumentParser
+from enum import StrEnum, auto
 from pathlib import Path
 
-ALL_ARCHITECTURES = [
-    'x86',
-    'x86-64',
-]
+
+class Architecture(StrEnum):
+    x86 = auto()
+    amd64 = auto()
+    arm64 = auto()
+
 
 MOD_PATCHES: dict[str, list[tuple[str, str]]] = {
     'acrylic-effect-radius-changer/1.1.0.wh.cpp': [
@@ -292,25 +295,46 @@
     'win7-alttab-loader': ['alttab.dll'],
 }
 
+SYMBOL_MODULES_SKIP_ARM64: dict[str, list[str]] = {
+    # comctl32.dll has duplicate symbols on ARM64.
+    "classic-desktop-icons": ['comctl32.dll'],
+    "classic-list-group-fix": ['comctl32.dll'],
+    "classic-taskdlg-fix": ['comctl32.dll'],
+    "dwm-unextend-frames": ['comctl32.dll'],
+    # comdlg32.dll has duplicate symbols on ARM64.
+    "classic-file-picker-dialog": ['comdlg32.dll'],
+}
+
 
 def get_mod_metadata(mod_source: str):
     p = r'^\/\/[ \t]+==WindhawkMod==[ \t]*$([\s\S]+?)^\/\/[ \t]+==\/WindhawkMod==[ \t]*$'
-    match = re.search(p, mod_source, re.MULTILINE)
+    match = re.search(p, mod_source, flags=re.MULTILINE)
     if not match:
         raise Exception(f'No metadata block')
 
     metadata_block = match.group(1)
 
     p = r'^\/\/[ \t]+@architecture[ \t]+(.*)$'
-    match = re.findall(p, metadata_block, re.MULTILINE)
-
-    architecture = match or ALL_ARCHITECTURES
-
-    if any (x not in ALL_ARCHITECTURES for x in architecture):
-        raise Exception(f'Unknown architecture')
+    match = re.findall(p, metadata_block, flags=re.MULTILINE)
+
+    architectures: set[Architecture] = set()
+
+    for arch in (match or ['x86', 'amd64', 'arm64']):
+        if arch == 'x86':
+            architectures.add(Architecture.x86)
+        elif arch == 'amd64':
+            architectures.add(Architecture.amd64)
+        elif arch == 'arm64':
+            architectures.add(Architecture.arm64)
+        elif arch == 'x86-64':
+            # Implies both amd64 and arm64.
+            architectures.add(Architecture.amd64)
+            architectures.add(Architecture.arm64)
+        else:
+            raise Exception(f'Unknown architecture: {arch}')
 
     return {
-        'architectures': architecture,
+        'architectures': architectures,
     }
 
 
@@ -368,7 +392,7 @@ def process_symbol_block(mod_source: str, symbol_block_match: re.Match, string_d
 
     # Make sure there are no preprocessor directives.
     p = r'^[ \t]*#.*'
-    if match := re.search(p, symbol_block, re.MULTILINE):
+    if match := re.search(p, symbol_block, flags=re.MULTILINE):
         raise Exception(f'Unsupported preprocessor directive: {match.group(0)}')
 
     # Merge strings spanning over multiple lines.
@@ -427,47 +451,69 @@ def sub_braced(match):
     }
 
 
-def get_mod_symbol_blocks(mod_source: str, arch: str):
-    # Expand #ifdef _WIN64 conditions.
-    def sub(match):
-        if match.group(1) in ['if', 'ifdef']:
-            condition_matches = arch == 'x86-64'
+def get_mod_symbol_blocks(mod_source: str, arch: Architecture):
+    # Expand #if architecture conditions.
+    def sub(sub_match):
+        condition1 = sub_match.group(1)
+        body1 = sub_match.group(2)
+        condition2 = sub_match.group(3)
+        body2 = sub_match.group(4)
+
+        if match := re.fullmatch(r'(?:if defined|ifdef|if)\b(.*)', condition1.strip()):
+            expression = match.group(1).strip()
+            negative = False
+        elif match := re.fullmatch(r'(?:|if !defined|ifndef|if !)\b(.*)', condition1.strip()):
+            expression = match.group(1).strip()
+            negative = True
         else:
-            assert match.group(1) == 'ifndef'
-            condition_matches = arch != 'x86-64'
+            raise Exception(f'Unsupported condition1: {condition1}')
+
+        if condition2 is not None and condition2.strip() != 'else':
+            raise Exception(f'Unsupported condition2: {condition2}')
+
+        if expression.startswith('(') and expression.endswith(')'):
+            expression = expression[1:-1].strip()
+
+        if expression == '_WIN64':
+            condition_matches = arch in [Architecture.amd64, Architecture.arm64]
+        elif expression == '_M_IX86':
+            condition_matches = arch == Architecture.x86
+        elif expression == '_M_X64':
+            condition_matches = arch == Architecture.amd64
+        elif expression == '_M_ARM64':
+            condition_matches = arch == Architecture.arm64
+        else:
+            # Not a supported arch condition, return as is.
+            return sub_match.group(0)
+
+        if negative:
+            condition_matches = not condition_matches
 
         if condition_matches:
-            return match.group(2)
+            return body1
 
-        return match.group(4) or ''
+        return body2 or ''
 
-    p = r'^[ \t]*#(if|ifn?def)[ \t]*_WIN64[ \t]*([\s\S]*?)(^[ \t]*#else[ \t]*$([\s\S]*?))?^[ \t]*#endif[ \t]*$'
+    p = r'^[ \t]*#\s*(if.*)$([\s\S]*?)(?:^[ \t]*#\s*(else.*)$([\s\S]*?))?^[ \t]*#endif[ \t]*$'
     mod_source = re.sub(p, sub, mod_source, flags=re.MULTILINE)
 
-    # Expand some #if defined(...) conditions.
-    def sub2(match):
-        return match.group(3) or ''
-
-    p = r'^[ \t]*#if defined\(_M_ARM64\)[ \t]*([\s\S]*?)(^[ \t]*#else[ \t]*$([\s\S]*?))?^[ \t]*#endif[ \t]*$'
-    mod_source = re.sub(p, sub2, mod_source, flags=re.MULTILINE)
-
     # Extract string definitions.
     p = r'^[ \t]*#[ \t]*define[ \t]+(\w+)[ \t]+L"(.*?)"[ \t]*$'
-    string_definitions = dict(re.findall(p, mod_source, re.MULTILINE))
+    string_definitions = dict(re.findall(p, mod_source, flags=re.MULTILINE))
     if any('"' in re.sub(r'\\.', '', x) for x in string_definitions.values()):
         raise Exception(f'Unsupported string definitions')
 
     # Extract symbol blocks.
     symbol_blocks = []
     p = r'^[ \t]*(?:const[ \t]+)?(?:CMWF_|WindhawkUtils::)?SYMBOL_HOOK[ \t]+(\w+)[\[ \t][\s\S]*?\};[ \t]*$'
-    for match in re.finditer(p, mod_source, re.MULTILINE):
+    for match in re.finditer(p, mod_source, flags=re.MULTILINE):
         symbol_block = process_symbol_block(mod_source, match, string_definitions)
         symbol_blocks.append(symbol_block)
 
     # Verify that no blocks were missed.
     p = r'SYMBOL_HOOK.*=(?![^{\n]+;)'
     if len(symbol_blocks) != len(
-        re.findall(p, remove_comments_from_code(mod_source), re.MULTILINE)
+        re.findall(p, remove_comments_from_code(mod_source), flags=re.MULTILINE)
     ):
         raise Exception(f'Unsupported symbol blocks')
 
@@ -542,6 +588,9 @@ def main():
                     if module in SYMBOL_MODULES_SKIP.get(mod_name, []):
                         continue
 
+                    if arch == Architecture.arm64 and module in SYMBOL_MODULES_SKIP_ARM64.get(mod_name, []):
+                        continue
+
                     result_arch = result.setdefault(mod_name, {}).setdefault(arch, {})
                     # Add unique symbols.
                     result_arch[module] = list(

scripts/02_download_binaries.py

@@ -73,6 +73,8 @@ def get_mapped_size(size):
 def download_binaries_from_symbol_server(name: str, target_folder: Path, previous_folder: Path, target_arch: str, insider=False):
     if insider:
         url = f'https://m417z.com/winbindex-data-insider/by_filename_compressed/{name}.json.gz'
+    elif target_arch == 'arm64':
+        url = f'https://m417z.com/winbindex-data-arm64/by_filename_compressed/{name}.json.gz'
     else:
         url = f'https://winbindex.m417z.com/data/by_filename_compressed/{name}.json.gz'
 
@@ -113,8 +115,10 @@ def download_binaries_from_symbol_server(name: str, target_folder: Path, previou
         if hash_file_info['machineType'] == 332:
             arch = 'x86'
         elif hash_file_info['machineType'] == 34404:
-            arch = 'x86-64'
-        elif hash_file_info['machineType'] in [452, 43620]:
+            arch = 'amd64'
+        elif hash_file_info['machineType'] == 43620:
+            arch = 'arm64'
+        elif hash_file_info['machineType'] in [452]:
             arch = str(hash_file_info['machineType'])
         else:
             raise Exception(f'Unknown machine type: {hash_file_info["machineType"]}')
@@ -212,6 +216,10 @@ def download_modules(module: tuple[str, str], binaries_folder: Path, previous_bi
 
     previous_folder = previous_binaries_folder / module_name / arch
 
+    # Temporary migration, TODO: remove.
+    if not previous_folder.exists() and arch == 'amd64':
+        previous_folder = previous_binaries_folder / module_name / 'x86-64'
+
     download_binaries_from_symbol_server(module_name, target_folder, previous_folder, arch)
     download_binaries_from_symbol_server(module_name, target_folder, previous_folder, arch, insider=True)
 

scripts/04_create_mod_cache.py

@@ -2,7 +2,7 @@
 from argparse import ArgumentParser
 from pathlib import Path
 
-MOD_CACHE_SEPARATORS = {
+MOD_CACHE_LEGACY_SEPARATORS = {
     'taskbar-button-click': '@',
     'taskbar-clock-customization': '@',
     'taskbar-thumbnail-reorder': '@',
@@ -85,27 +85,35 @@ def create_mod_cache_for_symbols_file(symbol_cache_path: Path,
             continue
 
         # Legacy symbol cache.
-        if arch == 'x86-64':
-            cache_key = f'symbol-cache-{binary_name}'
+        if arch == 'amd64':
+            legacy_cache_key = f'symbol-cache-{binary_name}'
+        elif arch == 'x86':
+            legacy_cache_key = f'symbol-{arch}-cache-{binary_name}'
         else:
-            cache_key = f'symbol-{arch}-cache-{binary_name}'
+            legacy_cache_key = None
 
-        symbol_cache_file_path = symbol_cache_path / mod_name / cache_key / f'{timestamp}-{image_size}.txt'
+        if legacy_cache_key:
+            symbol_cache_file_path = symbol_cache_path / mod_name / legacy_cache_key / f'{timestamp}-{image_size}.txt'
 
-        sep = MOD_CACHE_SEPARATORS.get(mod_name, '#')
+            sep = MOD_CACHE_LEGACY_SEPARATORS.get(mod_name, '#')
 
-        create_mod_cache_file(
-            symbol_cache_file_path,
-            sep,
-            str(timestamp),
-            str(image_size),
-            mod_archs[arch][binary_name],
-            symbols,
-        )
+            create_mod_cache_file(
+                symbol_cache_file_path,
+                sep,
+                str(timestamp),
+                str(image_size),
+                mod_archs[arch][binary_name],
+                symbols,
+            )
 
         # New symbol cache.
         if pdb_fingerprint is None:
-            cache_key = f'pe_{arch}_{timestamp}_{image_size}_{binary_name}'
+            arch_mapping = {
+                'x86': 'x86',
+                'amd64': 'x86-64',
+                'arm64': 'arm64',
+            }
+            cache_key = f'pe_{arch_mapping[arch]}_{timestamp}_{image_size}_{binary_name}'
         else:
             cache_key = f'pdb_{pdb_fingerprint}'
 
@@ -139,11 +147,17 @@ def create_mod_cache(binaries_folder: Path,
                 if not symbols_file.is_file():
                     continue
 
-                create_mod_cache_for_symbols_file(symbol_cache_path,
-                                                  extracted_symbols,
-                                                  binary_name.name,
-                                                  arch.name,
-                                                  symbols_file)
+                try:
+                    create_mod_cache_for_symbols_file(
+                        symbol_cache_path,
+                        extracted_symbols,
+                        binary_name.name,
+                        arch.name,
+                        symbols_file,
+                    )
+                except Exception as e:
+                    print(f'Failed to create mod cache for {symbols_file}: {e}')
+                    raise
 
 
 def main():