[tools] Use ThreadPool for docs generator

tools/scripts/docs_modm_io_generator.py

@@ -19,7 +19,8 @@
 import argparse
 import datetime
 import platform
-import multiprocessing as mp
+import subprocess
+from multiprocessing.pool import ThreadPool
 from pathlib import Path
 from jinja2 import Environment, FileSystemLoader
 from collections import defaultdict
@@ -101,13 +102,18 @@ def main():
     test_group.add_argument("--test2", "-t2", action='store_true', help="Test mode: generate only a few targets. List has targets from the real target list.")
     parser.add_argument("--jobs", "-j", type=int, default=2, help="Number of parallel doxygen processes")
     parser.add_argument("--local-temp", "-l", action='store_true', help="Create temporary directory inside current working directory")
-    group = parser.add_mutually_exclusive_group(required=True)
+    group = parser.add_mutually_exclusive_group()
     group.add_argument("--compress", "-c", action='store_true', help="Compress output into gzip archive")
     group.add_argument("--output", "-o", type=str, help="Output directory")
     parser.add_argument("--overwrite", "-f", action='store_true', help="Overwrite existing data in output directory (Removes all files from output directory.)")
     parser.add_argument("--deduplicate", "-d", action='store_true', help="Deduplicate identical files with symlinks.")
+    parser.add_argument("--target-job", help="Create a single target from job string.")
     args = parser.parse_args()
 
+    # Called by the thread pool below as a workaround for buggy multiprocessing
+    if args.target_job:
+        exit(0 if create_target(args.target_job) else -1)
+
     device_list, board_list, family_list = get_targets()
     if args.test:
         device_list = ["stm32f103c8t6", "stm32f417zgt6"] + family_list
@@ -136,11 +142,11 @@ def main():
         print("Starting to generate documentation...")
         template_overview(output_dir, device_list, board_list, template_path)
         print("... for {} devices, estimated memory footprint is {} MB".format(len(device_list) + len(board_list), (len(device_list)*70)+2000))
-        with mp.get_context("spawn").Pool(args.jobs) as pool:
+        with ThreadPool(args.jobs) as pool:
             # We can only pass one argument to pool.map
-            devices = ["{}|{}|{}||{}".format(modm_path, tempdir, dev, args.deduplicate) for dev in device_list]
-            devices += ["{}|{}|{}|{}|{}".format(modm_path, tempdir, dev, brd, args.deduplicate) for (brd, dev) in board_list]
-            results = pool.map(create_target, devices)
+            devices = [f"python3 {os.path.abspath(__file__)} --target-job '{modm_path}|{tempdir}|{dev}||{args.deduplicate}'" for dev in device_list]
+            devices += [f"python3 {os.path.abspath(__file__)} --target-job '{modm_path}|{tempdir}|{dev}|{brd}|{args.deduplicate}'" for (brd, dev) in board_list]
+            results = pool.map(lambda d: subprocess.run(d, shell=True).returncode, devices)
         # output_dir.rename(cwd / 'modm-api-docs')
         if args.compress:
             print("Zipping docs ...")
@@ -158,7 +164,7 @@ def main():
             print('Moving {} -> {}'.format(output_dir, final_output_dir))
             #shutil.move(str(output_dir) + '/', str(final_output_dir))
             output_dir.rename(final_output_dir)
-        return results.count(False)
+        return results.count(0) == len(results)
 
 
 def create_target(argument):
@@ -258,4 +264,4 @@ def template_overview(output_dir, device_list, board_list, template_path):
 
 
 if __name__ == "__main__":
-    exit(main())
+    exit(0 if main() else -1)