Merge pull request #1617 from heinezen/feature/export_threaded

Multi-threaded media export

openage/cabextract/cab.py

@@ -1,4 +1,4 @@
-# Copyright 2015-2023 the openage authors. See copying.md for legal info.
+# Copyright 2015-2024 the openage authors. See copying.md for legal info.
 
 """
 Provides CABFile, an extractor for the MSCAB format.
@@ -18,7 +18,7 @@
 from ..util.filelike.readonly import PosSavingReadOnlyFileLikeObject
 from ..util.filelike.stream import StreamFragment
 from ..util.files import read_guaranteed, read_nullterminated_string
-from ..util.fslike.filecollection import FileCollection
+from ..util.fslike.filecollection import FileCollection, FileEntry
 from ..util.math import INF
 from ..util.strings import try_decode
 from ..util.struct import NamedStruct, Flags
@@ -216,6 +216,28 @@ def verify_checksum(self) -> Union[None, NoReturn]:
             raise ValueError("checksum error in MSCAB data block")
 
 
+class CABEntry(FileEntry):
+    """
+    Entry in a CAB file.
+    """
+
+    def __init__(self, fileobj: CFFile):
+        self.fileobj = fileobj
+
+    def open_r(self):
+        return StreamFragment(
+            self.fileobj.folder.plain_stream,
+            self.fileobj.pos,
+            self.fileobj.size
+        )
+
+    def size(self) -> int:
+        return self.fileobj.size
+
+    def mtime(self) -> float:
+        return self.fileobj.timestamp
+
+
 class CABFile(FileCollection):
     """
     The actual file system-like CAB object.
@@ -275,20 +297,9 @@ def __init__(self, cab: FileLikeObject, offset: int = 0):
                     "CABFile has multiple entries with the same path: " +
                     b'/'.join(fileobj.path).decode())
 
-            def open_r(fileobj=fileobj):
-                """ Returns a opened ('rb') file-like object for fileobj. """
-                return StreamFragment(
-                    fileobj.folder.plain_stream,
-                    fileobj.pos,
-                    fileobj.size
-                )
-
-            self.add_fileentry(fileobj.path, (
-                open_r,
-                None,
-                lambda fileobj=fileobj: fileobj.size,
-                lambda fileobj=fileobj: fileobj.timestamp
-            ))
+            file_entry = CABEntry(fileobj)
+
+            self.add_fileentry(fileobj.path, file_entry)
 
     def __repr__(self):
         return "CABFile"

openage/convert/entity_object/export/texture.py

@@ -1,4 +1,4 @@
-# Copyright 2014-2023 the openage authors. See copying.md for legal info.
+# Copyright 2014-2024 the openage authors. See copying.md for legal info.
 
 """ Routines for texture generation etc """
 
@@ -14,7 +14,6 @@
 from ....log import spam
 from ...value_object.read.media.blendomatic import BlendingMode
 from ...value_object.read.media.hardcoded.terrain_tile_size import TILE_HALFSIZE
-from ...value_object.read.genie_structure import GenieStructure
 
 if typing.TYPE_CHECKING:
     from openage.convert.value_object.read.media.colortable import ColorTable
@@ -65,17 +64,13 @@ def get_data(self) -> numpy.ndarray:
         return self.data
 
 
-class Texture(GenieStructure):
-    image_format = "png"
+class Texture:
+    """
+    one sprite, as part of a texture atlas.
 
-    name_struct = "subtexture"
-    name_struct_file = "texture"
-    struct_description = (
-        "one sprite, as part of a texture atlas.\n"
-        "\n"
-        "this struct stores information about positions and sizes\n"
-        "of sprites included in the 'big texture'."
-    )
+    stores information about positions and sizes
+    of sprites included in the 'big texture'.
+    """
 
     def __init__(
         self,
@@ -180,18 +175,3 @@ def get_cache_params(self) -> tuple[tuple, tuple]:
             - PNG compression parameters (compression level + deflate params)
         """
         return self.best_packer_hints, self.best_compr
-
-    @classmethod
-    def get_data_format_members(cls, game_version) -> tuple:
-        """
-        Return the members in this struct.
-        """
-        data_format = (
-            (True, "x", None, "int32_t"),
-            (True, "y", None, "int32_t"),
-            (True, "w", None, "int32_t"),
-            (True, "h", None, "int32_t"),
-            (True, "cx", None, "int32_t"),
-            (True, "cy", None, "int32_t"),
-        )
-        return data_format

openage/convert/main.py

@@ -1,4 +1,4 @@
-# Copyright 2015-2023 the openage authors. See copying.md for legal info.
+# Copyright 2015-2024 the openage authors. See copying.md for legal info.
 #
 # pylint: disable=too-many-branches
 """
@@ -53,6 +53,10 @@ def convert_assets(
     if "compression_level" not in vars(args):
         args.compression_level = 1
 
+    # Set worker count for multi-threading if it was not set
+    if "jobs" not in vars(args):
+        args.jobs = None
+
     # Set verbosity for debug output
     if "debug_info" not in vars(args) or not args.debug_info:
         if args.devmode:
@@ -64,7 +68,7 @@ def convert_assets(
     # add a dir for debug info
     debug_log_path = converted_path / "debug" / datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
     debugdir = DirectoryCreator(debug_log_path).root
-    args.debugdir = AccessSynchronizer(debugdir).root
+    args.debugdir = debugdir
 
     # Create CLI args info
     debug_cli_args(args.debugdir, args.debug_info, args)
@@ -93,9 +97,8 @@ def convert_assets(
     if not data_dir:
         return None
 
-    # make srcdir and targetdir safe for threaded conversion
     args.srcdir = AccessSynchronizer(data_dir).root
-    args.targetdir = AccessSynchronizer(targetdir).root
+    args.targetdir = targetdir
 
     # Create mountpoint info
     debug_mounts(args.debugdir, args.debug_info, args)