[archive] add 'archive-mode' option (#5255)

docs/configuration.rst

@@ -844,6 +844,22 @@ Description
     An alternative `format string`_ to build archive IDs with.
 
 
+extractor.*.archive-mode
+------------------------
+Type
+    ``string``
+Default
+    ``"file"``
+Description
+    Controls when to write `archive IDs <extractor.*.archive-format_>`__
+    to the archive database.
+
+    * ``"file"``: Write IDs immediately
+      after completing or skipping a file download.
+    * ``"memory"``: Keep IDs in memory
+      and only write them after successful job completion.
+
+
 extractor.*.archive-prefix
 --------------------------
 Type
@@ -6172,7 +6188,7 @@ Description
 
     * format
         * General format string for logging messages
-          or a dictionary with format strings for each loglevel.
+          or an ``object`` with format strings for each loglevel.
 
           In addition to the default
           `LogRecord attributes <https://docs.python.org/3/library/logging.html#logrecord-attributes>`__,

gallery_dl/job.py

@@ -436,6 +436,8 @@ def handle_queue(self, url, kwdict):
 
     def handle_finalize(self):
         if self.archive:
+            if not self.status:
+                self.archive.finalize()
             self.archive.close()
 
         pathfmt = self.pathfmt
@@ -530,7 +532,11 @@ def initialize(self, kwdict=None):
                 if "{" in archive_path:
                     archive_path = formatter.parse(
                         archive_path).format_map(kwdict)
-                self.archive = archive.DownloadArchive(
+                if cfg("archive-mode") == "memory":
+                    archive_cls = archive.DownloadArchiveMemory
+                else:
+                    archive_cls = archive.DownloadArchive
+                self.archive = archive_cls(
                     archive_path, archive_format, archive_pragma)
             except Exception as exc:
                 extr.log.warning(