[python] Fix multiprocessing pool on Python 3.8

Processes are now spawned instead of forked, which can cause lockups and
lost processes

.github/workflows/compile-all.yml

@@ -20,7 +20,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Compile HAL for all AVRs
         run: |
           (cd test/all && python3 run_all.py at --quick-remaining)
@@ -46,7 +46,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Compile HAL for SAMD0x, SAMD1x, SAMD2x
         run: |
           (cd test/all && python3 run_all.py samd0 samd1 samd2 --quick-remaining)
@@ -72,7 +72,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Compile HAL for SAMD5x, SAME5x, SAMG5x
         run: |
           (cd test/all && python3 run_all.py samd5 same5 samg5 --quick-remaining)
@@ -98,7 +98,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Compile HAL for SAME7x, SAMS7x, SAMV7x
         run: |
           (cd test/all && python3 run_all.py same7 sams7 samv7 --quick-remaining)
@@ -124,7 +124,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Compile HAL for all STM32F0
         run: |
           (cd test/all && python3 run_all.py stm32f0 --quick-remaining)
@@ -150,7 +150,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Compile HAL for all STM32F1
         run: |
           (cd test/all && python3 run_all.py stm32f1 --quick-remaining)
@@ -176,7 +176,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Compile HAL for all STM32F2
         run: |
           (cd test/all && python3 run_all.py stm32f2 --quick-remaining)
@@ -202,7 +202,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Compile HAL for all STM32F3
         run: |
           (cd test/all && python3 run_all.py stm32f3 --quick-remaining)
@@ -228,7 +228,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Compile HAL for all STM32F4 Part 1
         run: |
           (cd test/all && python3 run_all.py stm32f4 --quick-remaining --split 3 --part 0)
@@ -254,7 +254,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Compile HAL for all STM32F4 Part 2
         run: |
           (cd test/all && python3 run_all.py stm32f4 --quick-remaining --split 3 --part 1)
@@ -280,7 +280,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Compile HAL for all STM32F4 Part 3
         run: |
           (cd test/all && python3 run_all.py stm32f4 --quick-remaining --split 3 --part 2)
@@ -306,7 +306,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Compile HAL for all STM32F7 Part 1
         run: |
           (cd test/all && python3 run_all.py stm32f7 --quick-remaining --split 2 --part 0)
@@ -332,7 +332,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Compile HAL for all STM32F7 Part 2
         run: |
           (cd test/all && python3 run_all.py stm32f7 --quick-remaining --split 2 --part 1)
@@ -358,7 +358,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Compile HAL for all STM32L0 Part 1
         run: |
           (cd test/all && python3 run_all.py stm32l0 --quick-remaining --split 2 --part 0)
@@ -384,7 +384,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Compile HAL for all STM32L0 Part 2
         run: |
           (cd test/all && python3 run_all.py stm32l0 --quick-remaining --split 2 --part 1)
@@ -411,7 +411,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Compile HAL for all STM32L1
         run: |
           (cd test/all && python3 run_all.py stm32l1 --quick-remaining)
@@ -437,7 +437,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Compile HAL for all STM32L4 Part 1
         run: |
           (cd test/all && python3 run_all.py stm32l4 --quick-remaining --split 3 --part 0)
@@ -463,7 +463,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Compile HAL for all STM32L4 Part 2
         run: |
           (cd test/all && python3 run_all.py stm32l4 --quick-remaining --split 3 --part 1)
@@ -489,7 +489,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Compile HAL for all STM32L4 Part 3
         run: |
           (cd test/all && python3 run_all.py stm32l4 --quick-remaining --split 3 --part 2)
@@ -515,7 +515,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Compile HAL for all STM32L5
         run: |
           (cd test/all && python3 run_all.py stm32l5 --quick-remaining)
@@ -541,7 +541,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Compile HAL for all STM32G0
         run: |
           (cd test/all && python3 run_all.py stm32g0 --quick-remaining)
@@ -567,7 +567,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Compile HAL for all STM32G4 Part 1
         run: |
           (cd test/all && python3 run_all.py stm32g4 --quick-remaining --split 2 --part 0)
@@ -593,7 +593,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Compile HAL for all STM32G4 Part 2
         run: |
           (cd test/all && python3 run_all.py stm32g4 --quick-remaining --split 2 --part 1)
@@ -619,7 +619,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Compile HAL for all STM32H7 Part 1
         run: |
           (cd test/all && python3 run_all.py stm32h7 --quick-remaining --split 2 --part 0)
@@ -645,7 +645,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Compile HAL for all STM32H7 Part 2
         run: |
           (cd test/all && python3 run_all.py stm32h7 --quick-remaining --split 2 --part 1)

.github/workflows/deploy-docs.yml

@@ -21,7 +21,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Mkdocs info
         run: |
           mkdocs --version
@@ -61,7 +61,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Full run of docs.modm.io-generator-script
         run: |
           export TERM=xterm-256color

.github/workflows/linux.yml

@@ -18,7 +18,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Check environment
         run: |
           env
@@ -129,7 +129,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Examples STM32F0 Series
         if: always()
         run: |
@@ -189,7 +189,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Examples STM32F4 Only Discovery Board
         if: always()
         run: |
@@ -209,7 +209,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Install protobuf==3.20.1 grpcio-tools (pip)
         run: |
           pip3 install --upgrade --upgrade-strategy=eager protobuf==3.20.1 grpcio-tools
@@ -232,7 +232,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Check environment
         run: |
           which avr-g++
@@ -278,7 +278,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Quick compile HAL for Cortex-M Part 1
         if: always()
         run: |
@@ -303,7 +303,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Quick compile HAL for Cortex-M Part 2
         if: always()
         run: |
@@ -328,7 +328,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Quick compile HAL for Cortex-M Part 3
         if: always()
         run: |
@@ -353,7 +353,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Quick compile HAL for Cortex-M Part 4
         if: always()
         run: |
@@ -378,7 +378,7 @@ jobs:
           git config --global --add safe.directory /__w/modm/modm
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
       - name: Install mkdocs mkdocs-material (pip)
         run: |
           pip3 install --upgrade --upgrade-strategy=eager mkdocs mkdocs-material

.github/workflows/macos.yml

@@ -63,7 +63,7 @@ jobs:
       - name: Update lbuild
         if: always()
         run: |
-          pip3 install --upgrade --user --upgrade-strategy=eager modm
+          pip3 install --upgrade --user --upgrade-strategy=eager modm multiprocess
 
       - name: Hosted Unittests
         if: always()

.github/workflows/windows_armcortexm.yml

@@ -50,7 +50,7 @@ jobs:
 
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
 
       - name: Compile STM32 Examples
         shell: bash

.github/workflows/windows_hosted.yml

@@ -52,7 +52,7 @@ jobs:
 
       - name: Update lbuild
         run: |
-          pip3 install --upgrade --upgrade-strategy=eager modm
+          pip3 install --upgrade --upgrade-strategy=eager modm multiprocess
 
       - name: Hosted Examples
         shell: bash

test/all/run_all.py

@@ -20,8 +20,9 @@
 import random
 import tempfile
 import argparse
+import platform
 import subprocess
-import multiprocessing
+import multiprocess
 from pathlib import Path
 from collections import defaultdict
 
@@ -94,7 +95,7 @@ def run_command(cmdline):
                            stderr=subprocess.PIPE, universal_newlines=True)
         return (p.stdout, p.stderr, p.returncode)
     except KeyboardInterrupt:
-        raise multiprocessing.ProcessError()
+        raise multiprocess.ProcessError()
 
 def run_lbuild(command):
     cmdline = ["lbuild"] + command
@@ -287,7 +288,7 @@ def build_device(run):
 
     print("Using {} parallel jobs for {} devices".format(args.jobs, len(devices)))
     try:
-        with multiprocessing.Pool(args.jobs) as pool:
+        with multiprocess.Pool(args.jobs) as pool:
             test_runs = pool.map(build_device,
                             [TestRun(x, cache_dir, cache_limit) for x in devices])