Add manage_containers.py

Dockerfile

@@ -23,7 +23,12 @@ RUN set -ex; \
       update-alternatives --install /usr/bin/arm-linux-gnueabi-g++ arm-linux-gnueabi-g++ /usr/bin/arm-linux-gnueabi-g++-${GCC_VERSION} 100; \
     fi; \
     if [ "$CLANG_VERSION" ]; then \
-      apt-get install -y -q clang-${CLANG_VERSION} lld-${CLANG_VERSION} clang-tools-${CLANG_VERSION}; \
+      if [ "$CLANG_VERSION" = "5" ] || [ "$CLANG_VERSION" = "6" ]; then \
+        CLANG_VERSION="${CLANG_VERSION}.0"; \
+        apt-get install -y -q clang-${CLANG_VERSION} lld-${CLANG_VERSION} clang-tools-6.0; \
+      else \
+        apt-get install -y -q clang-${CLANG_VERSION} lld-${CLANG_VERSION} clang-tools-${CLANG_VERSION}; \
+      fi; \
       update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${CLANG_VERSION} 100; \
       update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-${CLANG_VERSION} 100; \
       update-alternatives --install /usr/bin/lld lld /usr/bin/lld-${CLANG_VERSION} 100; \

README.md

@@ -37,6 +37,13 @@ __Supported gcc versions:__
  - gcc-14
 
 __Supported clang versions:__
+ - clang-5
+ - clang-6
+ - clang-7
+ - clang-8
+ - clang-9
+ - clang-10
+ - clang-11
  - clang-12
  - clang-13
  - clang-14
@@ -75,6 +82,125 @@ kernel-build-container   gcc-5      ed6270a767c6   8 hours ago          720MB
 kernel-build-container   gcc-4.9    408db89527ec   8 hours ago          913MB
 ```
 
+### Building containers with manage_containers.py:
+
+Get help:
+
+```console
+$ python3 manage_containers.py -h
+usage: manage_containers.py [-h] [-l] [-a compiler] [-r] [-q]
+
+Manage the kernel-build-containers
+
+options:
+  -h, --help            show this help message and exit
+  -l, --list            show the kernel build containers
+  -a compiler, --add compiler
+                        build a container with: (gcc-4.9, gcc-5, gcc-6, gcc-7, gcc-8, gcc-9, gcc-10, gcc-11, gcc-12, gcc-13, gcc-14, clang-5, clang-6, clang-7, clang-8, clang-9,
+                        clang-10, clang-11, clang-12, clang-13, clang-14, clang-15, clang-16, clang-17, all,where 'all' for all of the compilers)
+  -r, --remove          remove all created containers
+  -q, --quiet           suppress container build output
+```
+
+Building all containers:
+
+```console
+$ python3 manage_containers.py -a all
+```
+
+Created containers:
+
+```console
+$ sudo docker image list | grep kernel-build-container
+REPOSITORY               TAG         IMAGE ID       CREATED          SIZE
+kernel-build-container   clang-17    f82a39f4f536   10 seconds ago   4.87GB
+kernel-build-container   gcc-14      f82a39f4f536   10 seconds ago   4.87GB
+kernel-build-container   clang-16    debeffa0b306   2 minutes ago    4.89GB
+kernel-build-container   clang-15    282ee1c70ef4   5 minutes ago    3.49GB
+kernel-build-container   gcc-13      282ee1c70ef4   5 minutes ago    3.49GB
+kernel-build-container   clang-14    65e5a9bfec0f   7 minutes ago    1.88GB
+kernel-build-container   gcc-12      65e5a9bfec0f   7 minutes ago    1.88GB
+kernel-build-container   clang-13    57e0584fe0d3   12 minutes ago   1.96GB
+kernel-build-container   clang-12    fbb06aca13c9   13 minutes ago   1.71GB
+kernel-build-container   gcc-11      fbb06aca13c9   13 minutes ago   1.71GB
+kernel-build-container   clang-11    b3a75c3fd9ad   14 minutes ago   1.62GB
+kernel-build-container   gcc-10      b3a75c3fd9ad   14 minutes ago   1.62GB
+kernel-build-container   clang-10    6401372cf5aa   15 minutes ago   1.39GB
+kernel-build-container   gcc-9       6401372cf5aa   15 minutes ago   1.39GB
+kernel-build-container   clang-9     6668ffd11788   17 minutes ago   1.55GB
+kernel-build-container   gcc-8       6668ffd11788   17 minutes ago   1.55GB
+kernel-build-container   clang-8     23935b3308af   18 minutes ago   1.4GB
+kernel-build-container   gcc-7       23935b3308af   18 minutes ago   1.4GB
+kernel-build-container   clang-7     a9cc66b33f3e   19 minutes ago   1.5GB
+kernel-build-container   gcc-6       a9cc66b33f3e   19 minutes ago   1.5GB
+kernel-build-container   clang-6     8485509afe41   21 minutes ago   1.33GB
+kernel-build-container   gcc-5       8485509afe41   21 minutes ago   1.33GB
+kernel-build-container   clang-5     939c94215f80   22 minutes ago   1.87GB
+kernel-build-container   gcc-4.9     939c94215f80   22 minutes ago   1.87GB
+```
+
+Building all containers quietly:
+
+```console
+$ python3 manage_containers.py -a all -q
+```
+
+last part of expected output:
+
+```console
+Ubuntu | GCC    | Clang  | Status
+----------------------------------
+16.04  | 4.9    | 5      | [+]   
+16.04  | 5      | 6      | [+]   
+18.04  | 6      | 7      | [+]   
+18.04  | 7      | 8      | [+]   
+20.04  | 8      | 9      | [+]   
+20.04  | 9      | 10     | [+]   
+20.04  | 10     | 11     | [+]   
+22.04  | 11     | 12     | [+]   
+22.04  | 12     | 13     | [+]   
+22.04  | 12     | 14     | [+]   
+23.04  | 13     | 15     | [+]   
+24.04  | 14     | 16     | [+]   
+24.04  | 14     | 17     | [+] 
+```
+
+Building the specified container:
+
+```console
+$ python3 manage_containers.py -a clang-10
+```
+
+Building the specified container quietly:
+
+```console
+$ python3 manage_containers.py -a clang-10 -q
+```
+
+expected output:
+
+```console
+We need to use sudo to run the Docker
+Adding Ubuntu-20.04 container with GCC-9 and Clang-10
+sha256:cc53be032fc2e7ed3942cf399c68e3bc91284d3362a280673afd94be4b2455b5
+
+Ubuntu | GCC    | Clang  | Status
+----------------------------------
+16.04  | 4.9    | 5      | [-]   
+16.04  | 5      | 6      | [-]   
+18.04  | 6      | 7      | [-]   
+18.04  | 7      | 8      | [-]   
+20.04  | 8      | 9      | [-]   
+20.04  | 9      | 10     | [+]   
+20.04  | 10     | 11     | [-]   
+22.04  | 11     | 12     | [-]   
+22.04  | 12     | 13     | [-]   
+22.04  | 12     | 14     | [-]   
+23.04  | 13     | 15     | [-]   
+24.04  | 14     | 16     | [-]   
+24.04  | 14     | 17     | [-]
+```
+
 ### Running a container
 
 Get help:
@@ -291,3 +417,63 @@ usage: finish_container.sh kill/nokill out_dir
 ```console
 $ bash rm_containers.sh
 ```
+
+### Removing created Docker images (manage_containers.py alternative with warning for running containers):
+
+```console
+$ python3 manage_containers.py -r
+```
+
+last part of expected output without running containers:
+
+```console
+$ python3 manage_containers.py -r
+We need to use sudo to run the Docker
+
+Ubuntu | GCC    | Clang  | Status
+----------------------------------
+16.04  | 4.9    | 5      | [-]   
+16.04  | 5      | 6      | [-]   
+18.04  | 6      | 7      | [-]   
+18.04  | 7      | 8      | [-]   
+20.04  | 8      | 9      | [-]   
+20.04  | 9      | 10     | [-]   
+20.04  | 10     | 11     | [-]   
+22.04  | 11     | 12     | [-]   
+22.04  | 12     | 13     | [-]   
+22.04  | 12     | 14     | [-]   
+23.04  | 13     | 15     | [-]   
+24.04  | 14     | 16     | [-]   
+24.04  | 14     | 17     | [-]
+```
+
+last part of expected output with running clang-12 container:
+```consoler
+You still have running containers, that can't be removed:
+ 148254296e5d   kernel-build-container:clang-12   "/bin/bash"   14 seconds ago   Up 14 seconds             determined_grothendieck
+
+
+Ubuntu | GCC    | Clang  | Status
+----------------------------------
+16.04  | 4.9    | 5      | [-]   
+16.04  | 5      | 6      | [-]   
+18.04  | 6      | 7      | [-]   
+18.04  | 7      | 8      | [-]   
+20.04  | 8      | 9      | [-]   
+20.04  | 9      | 10     | [-]   
+20.04  | 10     | 11     | [-]   
+22.04  | 11     | 12     | [+]   
+22.04  | 12     | 13     | [-]   
+22.04  | 12     | 14     | [-]   
+23.04  | 13     | 15     | [-]   
+24.04  | 14     | 16     | [-]   
+24.04  | 14     | 17     | [-]
+```
+
+### Running tets for manage_containers.py:
+
+```console
+$ bash cov.sh
+```
+
+Results will be stored in htmlcov/index.html

cov.sh

@@ -0,0 +1,123 @@
+#!/bin/sh
+
+set -x
+set -e
+
+echo "Tests must be run with and without the docker group"
+
+if groups $USER | grep -q 'docker'; then
+    echo "Running tests with the docker group"
+else
+    echo "Running tests without the docker group"
+fi
+
+sleep 3 #sleep to notice the current group
+
+echo "Clearing coverage cache"
+coverage erase
+
+echo "Testing the help flag..."
+coverage run -a --branch manage_containers.py -h
+
+echo "Testing addition..."
+coverage run -a --branch manage_containers.py -a gcc-5
+coverage run -a --branch manage_containers.py -a gcc-6
+
+echo "Testing quiet addition..."
+coverage run -a --branch manage_containers.py -a gcc-7 -q
+coverage run -a --branch manage_containers.py -a gcc-8 -q
+
+echo "Testing removal..."
+coverage run -a --branch manage_containers.py -r
+coverage run -a --branch manage_containers.py -a gcc-4.9
+coverage run -a --branch manage_containers.py -a gcc-5
+coverage run -a --branch manage_containers.py -a gcc-6
+coverage run -a --branch manage_containers.py -a gcc-7
+coverage run -a --branch manage_containers.py -a gcc-8
+coverage run -a --branch manage_containers.py -a gcc-9
+coverage run -a --branch manage_containers.py -a gcc-10
+coverage run -a --branch manage_containers.py -a gcc-11
+coverage run -a --branch manage_containers.py -a gcc-12
+coverage run -a --branch manage_containers.py -a gcc-13
+coverage run -a --branch manage_containers.py -a gcc-14
+coverage run -a --branch manage_containers.py -r
+coverage run -a --branch manage_containers.py -a clang-7
+coverage run -a --branch manage_containers.py -a clang-8
+coverage run -a --branch manage_containers.py -a clang-9
+coverage run -a --branch manage_containers.py -a clang-10
+coverage run -a --branch manage_containers.py -a clang-11
+coverage run -a --branch manage_containers.py -a clang-12
+coverage run -a --branch manage_containers.py -a clang-13
+coverage run -a --branch manage_containers.py -a clang-14
+coverage run -a --branch manage_containers.py -a clang-15
+coverage run -a --branch manage_containers.py -a clang-16
+coverage run -a --branch manage_containers.py -a clang-17
+coverage run -a --branch manage_containers.py -r
+coverage run -a --branch manage_containers.py -a all
+coverage run -a --branch manage_containers.py -r
+coverage run -a --branch manage_containers.py -a all -q
+coverage run -a --branch manage_containers.py -r
+
+echo "Testing the list option..."
+coverage run -a --branch manage_containers.py -a gcc-12
+coverage run -a --branch manage_containers.py -l
+coverage run -a --branch manage_containers.py -r
+coverage run -a --branch manage_containers.py -l
+coverage run -a --branch manage_containers.py -a all
+coverage run -a --branch manage_containers.py -l
+coverage run -a --branch manage_containers.py -r
+coverage run -a --branch manage_containers.py -l
+
+echo "Testing removal with running containers..."
+coverage run -a --branch manage_containers.py -a gcc-12
+if groups $USER | grep 'docker'; then
+    docker run -d --rm --name test-running kernel-build-container:gcc-12 tail -f /dev/null
+    coverage run -a --branch manage_containers.py -r
+    docker stop test-running
+else
+    sudo docker run -d --rm --name test-running kernel-build-container:gcc-12 tail -f /dev/null
+    coverage run -a --branch manage_containers.py -r
+    sudo docker stop test-running
+fi
+coverage run -a --branch manage_containers.py -r
+
+echo "Collecting coverage for error handling"
+
+echo "Testing without any options..."
+coverage run -a --branch manage_containers.py && exit 1
+
+echo "Testing invalid arguments..."
+coverage run -a --branch manage_containers.py -a non-existent-compiler && exit 1
+coverage run -a --branch manage_containers.py -r invalid-option && exit 1
+
+echo "Testing invalid combinations..."
+coverage run -a --branch manage_containers.py -a gcc-10 -r && exit 1
+coverage run -a --branch manage_containers.py -a gcc-10 -r gcc-10 && exit 1
+coverage run -a --branch manage_containers.py -r gcc-10 && exit 1
+coverage run -a --branch manage_containers.py -r all && exit 1
+coverage run -a --branch manage_containers.py -a all -r && exit 1
+coverage run -a --branch manage_containers.py -a all -l && exit 1
+coverage run -a --branch manage_containers.py -r all -l && exit 1
+coverage run -a --branch manage_containers.py -a all -r -l && exit 1
+coverage run -a --branch manage_containers.py -a all -r -l -q && exit 1
+
+echo "Testing adding an existing container..."
+coverage run -a --branch manage_containers.py -a gcc-10
+coverage run -a --branch manage_containers.py -a gcc-10 && exit 1
+coverage run -a --branch manage_containers.py -a gcc-12
+coverage run -a --branch manage_containers.py -a clang-13 && exit 1 #it's the same container as gcc-12
+coverage run -a --branch manage_containers.py -a all
+coverage run -a --branch manage_containers.py -r
+
+echo "Testing invalid GCC/Clang version..."
+coverage run -a --branch manage_containers.py -a gcc-invalid && exit 1 
+coverage run -a --branch manage_containers.py -a clang-invalid && exit 1 
+
+echo "Testing unknown options..."
+coverage run -a --branch manage_containers.py --unknown-flag && exit 1 
+coverage run -a --branch manage_containers.py -a gcc-4.9 --unknown-flag && exit 1 
+
+echo "All tests completed. Creating report"
+coverage report --omit='/usr/lib/python3/dist-packages/*'
+coverage html --omit='/usr/lib/python3/dist-packages/*'
+rm .coverage

make_linux.py

@@ -13,7 +13,9 @@
 supported_archs = ['x86_64', 'i386', 'arm64', 'arm']
 supported_compilers = ['gcc-4.9', 'gcc-5', 'gcc-6', 'gcc-7', 'gcc-8', 'gcc-9',
                        'gcc-10', 'gcc-11', 'gcc-12', 'gcc-13', 'gcc-14',
-                       'clang-12', 'clang-13', 'clang-14', 'clang-15', 'clang-16', 'clang-17',
+                       'clang-5.0', 'clang-6.0', 'clang-7', 'clang-8',
+                       'clang-9', 'clang-10', 'clang-11', 'clang-12',
+                       'clang-13', 'clang-14', 'clang-15', 'clang-16', 'clang-17',
                        'all']
 
 NAME_DELIMITER = '__'