From db27fd3d52d3975a6604935bd30f7fcace09af01 Mon Sep 17 00:00:00 2001 From: Douglas Schilling Landgraf Date: Sun, 14 Jul 2024 10:13:39 -0400 Subject: [PATCH] demo devconf: initial setup for tests Signed-off-by: Douglas Schilling Landgraf Signed-off-by: Yariv Rachmani --- demos/devconf-2024/lib/ContainerFile.template | 130 ++++++++ demos/devconf-2024/lib/container | 247 +++++++++++++++ .../lib/quadlet/container-template.container | 11 + demos/devconf-2024/lib/systemd | 84 +++++ demos/devconf-2024/lib/tests | 55 ++++ demos/devconf-2024/lib/utils | 81 +++++ demos/devconf-2024/setup | 298 ++++++++++++++++++ demos/devconf-2024/tools/FFI/.gitignore | 1 + demos/devconf-2024/tools/FFI/Makefile | 38 +++ demos/devconf-2024/tools/FFI/README.md | 27 ++ .../tools/FFI/deny_sched_setattr/README.md | 27 ++ .../execute_sched_setattr.c | 91 ++++++ .../tools/FFI/deny_set_scheduler/README.md | 22 ++ .../execute_set_scheduler.c | 28 ++ .../tools/FFI/disk/QM/file-allocate.c | 43 +++ .../tools/FFI/memory/memory_eat.c | 49 +++ demos/devconf-2024/tools/FFI/module/README.md | 21 ++ .../tools/FFI/module/modprobe_module | 19 ++ demos/devconf-2024/tools/FFI/sysctl/README.md | 26 ++ demos/devconf-2024/tools/FFI/sysctl/setsysctl | 35 ++ demos/devconf-2024/tools/README.md | 5 + demos/devconf-2024/tools/remove-containers | 15 + 22 files changed, 1353 insertions(+) create mode 100644 demos/devconf-2024/lib/ContainerFile.template create mode 100644 demos/devconf-2024/lib/container create mode 100644 demos/devconf-2024/lib/quadlet/container-template.container create mode 100644 demos/devconf-2024/lib/systemd create mode 100644 demos/devconf-2024/lib/tests create mode 100644 demos/devconf-2024/lib/utils create mode 100755 demos/devconf-2024/setup create mode 100644 demos/devconf-2024/tools/FFI/.gitignore create mode 100644 demos/devconf-2024/tools/FFI/Makefile create mode 100644 demos/devconf-2024/tools/FFI/README.md create mode 100644 demos/devconf-2024/tools/FFI/deny_sched_setattr/README.md create mode 100644 demos/devconf-2024/tools/FFI/deny_sched_setattr/execute_sched_setattr.c create mode 100644 demos/devconf-2024/tools/FFI/deny_set_scheduler/README.md create mode 100644 demos/devconf-2024/tools/FFI/deny_set_scheduler/execute_set_scheduler.c create mode 100644 demos/devconf-2024/tools/FFI/disk/QM/file-allocate.c create mode 100644 demos/devconf-2024/tools/FFI/memory/memory_eat.c create mode 100644 demos/devconf-2024/tools/FFI/module/README.md create mode 100644 demos/devconf-2024/tools/FFI/module/modprobe_module create mode 100644 demos/devconf-2024/tools/FFI/sysctl/README.md create mode 100755 demos/devconf-2024/tools/FFI/sysctl/setsysctl create mode 100644 demos/devconf-2024/tools/README.md create mode 100755 demos/devconf-2024/tools/remove-containers diff --git a/demos/devconf-2024/lib/ContainerFile.template b/demos/devconf-2024/lib/ContainerFile.template new file mode 100644 index 00000000..93635baa --- /dev/null +++ b/demos/devconf-2024/lib/ContainerFile.template @@ -0,0 +1,130 @@ +FROM quay.io/centos/centos:stream9 + +RUN dnf install -y \ + dnf-plugin-config-manager \ + epel-release + +RUN dnf config-manager -y --set-enabled crb + +RUN dnf install -y \ + clang-tools-extra \ + codespell \ + bzip2 \ + cargo \ + rust \ + gcc \ + dbus-devel \ + g++ \ + go-toolset \ + fpaste \ + lcov \ + hostname \ + podman \ + python3-podman \ + python3-flake8 \ + python3-pytest \ + python3-pytest-timeout \ + python3-dasbus \ + pre-commit \ + pcp \ + sudo \ + gpgme-devel \ + libseccomp-devel \ + iptables-nft \ + iptables-services \ + selinux-policy-targeted \ + selinux-policy-devel \ + ShellCheck \ + systemd \ + valgrind \ + createrepo_c \ + dnf-utils \ + git \ + gzip \ + jq \ + tmt \ + container-selinux \ + kernel \ + kernel-modules \ + passwd \ + python3-gobject \ + python3-pip \ + meson \ + npm \ + rpm-build \ + ruby \ + ruby-devel \ + sed \ + vim-enhanced \ + systemd-devel \ + iputils \ + tar \ + golang-github-cpuguy83-md2man \ + telnet \ + net-tools \ + iproute + +WORKDIR /root + +RUN gem install ruby-dbus +RUN dnf -y copr enable @centos-automotive-sig/bluechi-snapshot centos-stream-9 +RUN dnf -y install bluechi bluechi-agent bluechi-ctl +RUN alternatives --install /usr/bin/python python /usr/bin/python3 1 +RUN npm install markdownlint-cli2 --global + +# Script will remove the lines before the commands automatically +# [manually install bluechi] RUN rpm -e bluechi bluechi-agent bluechi-ctl bluechi-selinux +# [manually install bluechi] RUN git clone @BUILD_BLUECHI_FROM_GH_URL@ +# [manually install bluechi] WORKDIR /root/bluechi + + +# Look for remote branches +# [manually install bluechi] RUN if git branch -a | grep "remotes/origin/@BRANCH_BLUECHI@" &> /dev/null; then \ +# [manually install bluechi] git checkout @BRANCH_BLUECHI@ &> /dev/null; \ +# [manually install bluechi] else \ +# Look to see if the branch already exists, like main branch +# [manually install bluechi] if git branch -a | grep -w @BRANCH_BLUECHI@ &> /dev/null; then \ +# [manually install bluechi] git checkout @BRANCH_BLUECHI@ || true &> /dev/null; \ +# The Branch doesn't exist, create one using -b +# [manually install bluechi] else \ +# [manually install bluechi] git checkout -b @BRANCH_BLUECHI@ || true &> /dev/null; \ +# [manually install bluechi] fi \ +# [manually install bluechi] fi + +# Execute the build after selecting the branch +# [manually install bluechi] RUN ./build-scripts/build-rpm.sh || true + +# Upgrade the current bluechi using the new rpm packages but exclude src.rpm +# [manually install bluechi] RUN find . -not -name *.src* -name *.rpm | xargs dnf install -y +# RUN cp /usr/share/bluechi/config/controller.conf /etc/bluechi/ +# [manually install bluechi] RUN cp /usr/share/bluechi/config/controller.conf /etc/bluechi/ + +# [start] CONTROL - settings +# Setting configuration, uncommenting: +# - ControllerPort to communicate via 842 +# - Use journal for logging +# - Set AllowedNodeNames as control and node1 +# TODO: remove the static node1 here, dynamic by rune2e script via sed +RUN cp /usr/share/bluechi/config/controller.conf /etc/bluechi/ +RUN cp /usr/share/bluechi-agent/config/agent.conf /etc/bluechi/agent.conf.d/ +RUN sed -e '/^#ControllerPort=/s/^#//g' \ + -e '/#LogTarget/s/^#//' \ + -e "s/^#AllowedNodeNames=/AllowedNodeNames=control,node1/" \ + -i /etc/bluechi/controller.conf +# [end] CONTROL - settings + +RUN echo 'Foobar!' | passwd --stdin root + +# [start] CONTROL - agent settings +RUN cp /usr/share/bluechi-agent/config/*.conf /etc/bluechi/agent.conf.d/ &> /dev/null +RUN IPv4_CONTROL="$(ip -brief address show eth0 | awk '{print $3}' | awk -F/ '{print $1}')" && \ + sed -e "s/^#NodeName=/NodeName=control/" \ + -i /etc/bluechi/agent.conf.d/agent.conf + +# [end] CONTROL - agent settings + +# [start] CONTROL - enable services and systemd +RUN systemctl enable bluechi-controller &> /dev/null +RUN systemctl enable bluechi-agent &> /dev/null +CMD ["/usr/lib/systemd/systemd"] +# [end] CONTROL - enable services and systemd diff --git a/demos/devconf-2024/lib/container b/demos/devconf-2024/lib/container new file mode 100644 index 00000000..14290acc --- /dev/null +++ b/demos/devconf-2024/lib/container @@ -0,0 +1,247 @@ +#!/bin/bash +# shellcheck disable=SC2129 +# +# Copyright 2023 The qm Authors +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; If not, see . + +get_ip_container() { + local container_name="${1}" + local container_interface="${2}" + local ip_output + + ip_output=$(podman exec \ + "${container_name}" \ + ip \ + --brief \ + address \ + show \ + "${container_interface}") || exit 1 + + echo "${ip_output}" | awk '{print $3}' | awk -F/ '{print $1}' +} + +create_container() { + local container_file="${1}" + local container_name="${2}" + local container_tag="${3}" + local container_add_caps="${4}" + local image_name + + image_name=$(podman images \ + "${container_tag}" -n + ) + if [ -z "${image_name}" ]; then + # Execute build with tag for latest + if [ -z "${container_add_caps}" ]; then + podman_build="podman build -q -f ${container_file} --tag ${container_tag} 1> /dev/null" + else + podman_build="podman build --cap-add ${container_add_caps} -q -f ${container_file} --tag ${container_tag} 1> /dev/null" + fi + eval "$podman_build" + if_error_exit "create_container: \ +podman build failed! \ +file: ${container_file} tag: ${container_tag}" + image_name=$(podman images \ + "${container_tag}" -n + ) + fi + IMAGE_ID=$(echo "${image_name}" | awk '{print $3}' | uniq) + + # Execute the container + eval "$(podman run -d \ + --privileged \ + --network podmanDualStack \ + --name "${container_name}" \ + --hostname "${container_name}" \ + "${IMAGE_ID}" 1> /dev/null + )" + if_error_exit "create_container: podman run failed! \ +name: ${container_name} \ +hostname: ${container_name} \ +imageID: ${IMAGE_ID}" +} + +setup_node() { + local nodeID="${1}" + local use_copr_repo="${2}" + + # copy control as template to node${number} + cp ./lib/ContainerFile.template ContainerFile.node"${nodeID}" + if_error_exit "failed to copy ContainerFile control template to node!" + if [ -n "${use_copr_repo}" ]; then + sed -e "/crb/a RUN dnf -y copr enable ${use_copr_repo} centos-stream-9" -i ContainerFile.node"${nodeID}" + if_error_exit "failed to sed ContainerFile node template!" + fi + # remove specific code from control mode (template) + sed '/start/,/end/d' -i ContainerFile.node"${nodeID}" + if_error_exit "failed to sed ContainerFile node template!" +} + +set_nodename_all_nodes() { + for nodeID in $(seq 1 "${NUMBER_OF_NODES}") ; + do + eval "$(podman exec node"${nodeID}" \ + sed -i 's/^#NodeName=/NodeName='node"${nodeID}"'/g' \ + /etc/bluechi/agent.conf.d/agent.conf + )" + if_error_exit "node: unable to sed NodeName in bluechi agent.conf" + + # restarting the qm bluechi-agent + eval "$(podman exec node"${nodeID}" \ + systemctl restart bluechi-agent + )" + if_error_exit "node: unable to restart bluechi-agent service" + done +} + +qm_set_nodename_all_nodes() { + for nodeID in $(seq 1 "${NUMBER_OF_NODES}") ; + do + qm_node_name="qm-node${nodeID}" + eval "$(podman exec node"${nodeID}" \ + podman exec qm \ + sed -i 's/^#NodeName=/NodeName='"${qm_node_name}"'/g' \ + /etc/bluechi/agent.conf.d/agent.conf + )" + if_error_exit "qm node: unable to sed NodeName in bluechi agent.conf" + + # restarting the qm bluechi-agent + eval "$(podman exec node"${nodeID}" \ + podman exec qm \ + systemctl restart bluechi-agent + )" + if_error_exit "qm node: unable to restart bluechi-agent service" + done +} + +create_qm_node() { + # Generates 'N' containers QM nodes as required for the test scale + + AllowedNodeNames="" + for nodeID in $(seq 1 "${NUMBER_OF_NODES}") ; + do + # Generates the Container.node${ID} + eval "IP_CONTROL_MACHINE=\$(get_ip_container \${CONTROL_CONTAINER_NAME} \${NET_INTERFACE_IP_CONTROL})" + if_error_exit "unable to get ip from ${CONTROL_CONTAINER_NAME}" + + setup_node \ + "${nodeID}" \ + "${USE_QM_COPR}" + # Add final instructions + echo "RUN dnf install qm -y &> /dev/null" >> ContainerFile.node"${nodeID}" + + if [ -n "${QM_GH_URL}" ]; then + curl "${QM_GH_URL}" > /usr/share/qm/setup + chmod +x /usr/share/qm/setup + fi + + # Execute qm setup + # Use --skip-systemctl true in podman build, systemd is not running + # Do not set --skip-systemctl inside running container with systemd in it. + echo "RUN /usr/share/qm/setup --skip-systemctl true 2>&1 > /tmp/qm-setup.log || echo "QM setup failed, please check /tmp/qm-setup.log."" >> ContainerFile.node"${nodeID}" + + # Enable bluechi-agent + echo 'RUN cp /usr/share/bluechi-agent/config/*.conf /etc/bluechi/agent.conf.d/' >> ContainerFile.node"${nodeID}" + echo 'RUN sed -i -e "s/^#ControllerHost=/ControllerHost='"${IP_CONTROL_MACHINE}/"\" \ + ' /etc/bluechi/agent.conf.d/agent.conf' >> ContainerFile.node"${nodeID}" + echo "RUN systemctl enable bluechi-agent &> /dev/null" >> ContainerFile.node"${nodeID}" + + # Add systemd as CMD + echo "CMD [\"/usr/lib/systemd/systemd\"]" >> ContainerFile.node"${nodeID}" + + # create the container ${nodeID} + info_message "Creating container \033[92mnode${nodeID}\033[0m [\033[92mQM mode\033[0m]" + + CONTAINER_ADD_CAPS="SYS_ADMIN /tmp/" + create_container \ + ContainerFile.node"${nodeID}" \ + "node${nodeID}" \ + "node:latest" \ + "${CONTAINER_ADD_CAPS}" + + # qm - after the setup, reload daemon and start qm + eval "$(podman exec \ + node"${nodeID}" \ + systemctl daemon-reload &> /dev/null + )" + if_error_exit "unable to execute systemctl daemon-load in node${nodeID}" + + eval "$(podman exec \ + node"${nodeID}" \ + systemctl start qm &> /dev/null + )" + if_error_exit "unable to execute systemctl start qm" + + # set in the QM container the ControllerHost (Control Machine) and restart bluechi-agent + eval "$(podman exec node"${nodeID}" \ + podman exec qm \ + cp \ + /usr/share/bluechi-agent/config/agent.conf \ + /etc/bluechi/agent.conf.d/agent.conf + )" + if_error_exit "unable to copy agent.conf template to agent.conf.d dir" + + qm_node_name="qm-node${nodeID}" + NODES_FOR_TESTING+=("${qm_node_name}") + + eval "$(podman exec node"${nodeID}" \ + sed -i -e 's/^#ControllerHost=.*/ControllerHost='"${IP_CONTROL_MACHINE}"'/g' \ + -e 's/^ControllerHost=.*/ControllerHost='"${IP_CONTROL_MACHINE}"'/g' \ + /etc/bluechi/agent.conf.d/agent.conf + )" + if_error_exit "node: unable to sed ControllerHost in bluechi agent.conf" + + # restarting the node bluechi-agent + eval "$(podman exec node"${nodeID}" \ + systemctl restart bluechi-agent + )" + if_error_exit "node: unable to restart bluechi-agent service" + + eval "$(podman exec node"${nodeID}" \ + podman exec qm \ + sed -i 's/^#ControllerHost=/ControllerHost='"${IP_CONTROL_MACHINE}"'/g' \ + /etc/bluechi/agent.conf.d/agent.conf + )" + if_error_exit "qm node: unable to sed ControllerHost in bluechi agent.conf" + + AllowedNodeNames="${AllowedNodeNames}\n ${qm_node_name},\n node${nodeID}," + done + + # Remove the last , from the string, otherwise bluechi won't understand + AllowedNodeNames="${AllowedNodeNames%?}" + + # CONTROL NODE: append QM node into /etc/bluechi/agent.conf.din the control node the new qm node name + eval "$(podman exec "${CONTROL_CONTAINER_NAME}" \ + sed -i '/^AllowedNodeNames=/ s/$/,'"${AllowedNodeNames}"'/' \ + /etc/bluechi/controller.conf + )" + if_error_exit "control node: unable to sed AllowedNodeName in controller.conf" + + # restart the bluechi-controller service + eval "$(podman exec \ + "${CONTROL_CONTAINER_NAME}" \ + systemctl restart bluechi-controller + )" + if_error_exit "control node: unable to restart bluechi-controller service" +} + +create_asil_node() { + # Creates the control container - a.k.a ASIL + info_message "Creating container \033[92m${CONTROL_CONTAINER_NAME}\033[0m [\033[92mASIL mode\033[0m]. It might take some time..." + create_container \ + "./lib/ContainerFile.template" \ + "${CONTROL_CONTAINER_NAME}" \ + "${TAG_CONTROL_MACHINE}" +} diff --git a/demos/devconf-2024/lib/quadlet/container-template.container b/demos/devconf-2024/lib/quadlet/container-template.container new file mode 100644 index 00000000..fb657010 --- /dev/null +++ b/demos/devconf-2024/lib/quadlet/container-template.container @@ -0,0 +1,11 @@ +[Unit] +Description=The sleep container +After=local-fs.target + +[Container] +Image=registry.access.redhat.com/ubi9-minimal:latest +Exec=sleep infinity + +[Install] +# Start by default on boot +WantedBy=multi-user.target default.target diff --git a/demos/devconf-2024/lib/systemd b/demos/devconf-2024/lib/systemd new file mode 100644 index 00000000..644fde39 --- /dev/null +++ b/demos/devconf-2024/lib/systemd @@ -0,0 +1,84 @@ +#!/bin/bash +# +# Copyright 2023 The qm Authors +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; If not, see . + +create_stub_systemd_srv() { + container_target="${1}" + creates_on_qm_node="${2}" + shift + + if [ -n "${creates_on_qm_node}" ]; then + command_on_qm_node="podman exec qm" # keep the space before podman + else + shift # remove the "" from qm param to the loop below + fi + services_to_be_stub=("$@") + + + for srv in "${services_to_be_stub[@]}" + do + if [ "${srv}" = "qm" ]; then + continue + fi + + # QM container + if [ -n "${creates_on_qm_node}" ]; then + info_message "Creating quadlet container file: \033[92mcontainer-${srv}\033[0m in \033[92m${container_target}\033[0m and moving it to container \033[92mqm\033[0m to start the quadlet service..." + cmd_cp_systemd_srv="podman \ +cp \ +lib/quadlet/container-template.container \ +${container_target}:/tmp/container-${srv}.container" + #echo "${cmd_cp_systemd_srv}" + eval "${cmd_cp_systemd_srv}" + if_error_exit "cannot copy container-${srv} to qm container" + + # Find configuration file from service + target_service_file=$(podman exec ${container_target} systemctl show -P SourcePath qm.service) + # START: remove DropCapability to run nested container + podman exec -it ${container_target} bash -c "sed -i '/^\s*DropCapability=sys_resource/ d' \"${target_service_file}\"" + if_error_exit "unable to remove DropCapability=sys_resource in qm.container" + + podman exec -it ${container_target} systemctl daemon-reload + if_error_exit "unable to execute daemon-reload" + + podman exec -it ${container_target} systemctl restart qm + if_error_exit "unable to restart qm service" + # END: remove DropCapability to run nested container + + # START: copy template quadlet to qm container + podman exec -it ${container_target} podman cp /tmp/container-${srv}.container qm:/etc/containers/systemd + if_error_exit "unable to copy container-${srv}.container to qm container" + + podman exec -it ${container_target} podman exec -it qm systemctl daemon-reload + if_error_exit "unable to execute on qm container systemctl daemon-reload" + + podman exec -it ${container_target} podman exec -it qm systemctl start container-${srv} + if_error_exit "unable to start the quadlet service container-${srv}" + + # END: copy template quadlet to qm container + # Not a QM container, example: bluechi-controller + else + info_message "Creating and starting quadlet container: \033[92mcontainer-${srv}\033[0m in \033[92m${container_target}\033[0m. It might take some time..." + cmd_cp_systemd_serv="podman \ +cp \ +lib/quadlet/container-template.container \ +${container_target}:/etc/containers/systemd/container-${srv}.container" + #echo "${cmd_cp_systemd_serv}" + eval "${cmd_cp_systemd_serv}" + if_error_exit "cannot copy container-${srv} to ${container_target} container" + fi + done +} diff --git a/demos/devconf-2024/lib/tests b/demos/devconf-2024/lib/tests new file mode 100644 index 00000000..d28a1654 --- /dev/null +++ b/demos/devconf-2024/lib/tests @@ -0,0 +1,55 @@ +#!/bin/bash +# +# shellcheck disable=SC2046,SC2116 +# +# Copyright 2023 The qm Authors +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; If not, see . +# + +NODES_FOR_TESTING_ARR="${NODES_FOR_TESTING_ARR:-control qm-node1}" +readarray -d ' ' -t NODES_FOR_TESTING <<< "$NODES_FOR_TESTING_ARR" +CONTROL_CONTAINER_NAME="${CONTROL_CONTAINER_NAME:-control}" +WAIT_BLUECHI_AGENT_CONNECT="${WAIT_BLUECHI_AGENT_CONNECT:-5}" + +test_bluechi_list_all_units() { + + local bluechictl_cmd + + for node_name in "${NODES_FOR_TESTING[@]}" + do + echo + info_message "Connected to \033[92m${CONTROL_CONTAINER_NAME}\033[0m, listing few systemd units from \033[92m${node_name}\033[0m" + if [ "${CONTROL_CONTAINER_NAME}" == "host" ]; then + bluechictl_cmd="bluechictl list-units ${node_name}" + else + # It could take some time for qm bluechi-agent to connect + if [[ "${node_name}" =~ .*qm.* ]];then + sleep "${WAIT_BLUECHI_AGENT_CONNECT}" + fi + bluechictl_cmd="podman exec" + bluechictl_cmd+=" ${CONTROL_CONTAINER_NAME}" + bluechictl_cmd+=" bluechictl list-units ${node_name}" + fi + cmd_result=$(eval "${bluechictl_cmd}") + + if_error_exit "unable to execute bluechictl command on ${CONTROL_CONTAINER_NAME}" + + if [ "${CONTROL_CONTAINER_NAME}" == "host" -o "${CONTROL_CONTAINER_NAME}" == "autosd" ]; then + grep -E "bluechi|qm" <<< ${cmd_result} + else + grep -E "container-|qm.service" <<< ${cmd_result} + fi + done +} diff --git a/demos/devconf-2024/lib/utils b/demos/devconf-2024/lib/utils new file mode 100644 index 00000000..df695ec4 --- /dev/null +++ b/demos/devconf-2024/lib/utils @@ -0,0 +1,81 @@ +#!/bin/bash +# +# Copyright 2023 The qm Authors +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; If not, see . + +if_error_exit() { + ########################################################################### + # Description: # + # Validate if previous command failed and show an error msg (if provided) # + # # + # Arguments: # + # $1 - error message if not provided, it will just exit # + ########################################################################### + local exit_code="$?" + if [ "${exit_code}" != "0" ]; then + RED="\033[91m" + ENDCOLOR="\033[0m" + echo -e "[ ${RED}FAILED${ENDCOLOR} ] ${1} with exit code: ${exit_code}" + kill $$ &> /dev/null + fi +} + +info_message() { + ########################################################################### + # Description: # + # show [INFO] in blue and a message as the validation passed. # + # # + # Arguments: # + # $1 - message to output # + ########################################################################### + if [ -z "${1}" ]; then + echo "info_message() requires a message" + exit 1 + fi + BLUE="\033[94m" + ENDCOLOR="\033[0m" + echo -e "[ ${BLUE}INFO${ENDCOLOR} ] ${1}" +} + +exec_cmd() { + local cmd="$1" + eval "$cmd" + if_error_exit "Error: Command $cmd failed" +} + +cleanup() { + info_message "Cleaning any existing artifacts that were generated during previous runs" + output=$(podman ps -a --format "{{.Names}}") + IFS=$'\n' # Set the internal field separator to newline + + # DO NOT use double quotes here + for node in ${output}; do + if [[ "${node}" == "control" ]] || [[ "${node}" =~ ^node.* ]]; then + info_message " - Removing container: ${node}" + # Make sure we remove previous settings + podman rm --storage --force "${node}" &> /dev/null + if_error_exit "error removing storage ${1}" + + podman rm "${node}" --force 1> /dev/null + if_error_exit "error removing container ${1}" + + rm -f ./*.node* + fi + done + info_message " - Removing podmanDualStack network..." + info_message " " + podman network rm podmanDualStack -f 1> /dev/null + +} diff --git a/demos/devconf-2024/setup b/demos/devconf-2024/setup new file mode 100755 index 00000000..0f3cdcee --- /dev/null +++ b/demos/devconf-2024/setup @@ -0,0 +1,298 @@ +#!/bin/bash + +set -eoux pipefail + +# shellcheck disable=SC1091 +# +# Copyright 2023 The qm Authors +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; If not, see . +# +# Capture the start time +START_TIME=$(date +%s) +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}"/lib/utils +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}"/lib/container +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}"/lib/systemd +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}"/lib/tests + +# GLOBALS +export CONFIG_NODE_AGENT_PATH="/etc/bluechi/agent.conf.d/agent.conf" +export REGISTRY_UBI8_MINIMAL="registry.access.redhat.com/ubi8/ubi-minimal" +export WAIT_BLUECHI_SERVER_BE_READY_IN_SEC=5 +export CONTROL_CONTAINER_NAME="control" +export NODES_FOR_TESTING=("control" "node1") +export IP_CONTROL_MACHINE="" +export CONTAINER_CAP_ADD="" +export ARCH="" + +export BUILD_BLUECHI_FROM_GH_URL="" +export QM_GH_URL="" +export BRANCH_QM="" +export SET_QM_PART="" +export USE_QM_COPR="${PACKIT_COPR_PROJECT:-rhcontainerbot/qm}" + +RED='\033[91m' +GRN='\033[92m' +CLR='\033[0m' + +# ====================== Start - int main {} ;-) +ARGUMENT_LIST=( + "qm-setup-from-gh-url" + "branch-qm" + "set-qm-disk-part" + "use-qm-copr" +) + +usage() { +cat < /usr/share/qm/setup + chmod +x /usr/share/qm/setup + fi + # Curl files into here, + # Fix: default setup:main should be removed on next qm release + /usr/share/qm/setup --hostname localrootfs + cat > /etc/bluechi/controller.conf << 'EOF' +[bluechi-controller] +AllowedNodeNames=qm.localrootfs,localrootfs +ControllerPort=842 +LogLevel=INFO +LogTarget=journald +LogIsQuiet=false +EOF +cat > /etc/bluechi/agent.conf.d/00-default.conf << 'EOF' +[bluechi-agent] +NodeName=localrootfs +EOF + # Enable services + info_message "Setup qm services, enable bluechi services" + info_message "==============================" + systemctl enable bluechi-controller + systemctl enable bluechi-agent + # Start services + info_message "Setup qm services, start bluechi services" + info_message "==============================" + systemctl start bluechi-controller + systemctl start bluechi-agent + # Restart qm to read lates bluechi-agent.conf + systemctl restart qm +} + +install_autosd_repo() { + # Install autosd repository + touch /etc/yum.repos.d/autosd.repo + cat > "/etc/yum.repos.d/autosd.repo" << EOF +[autosd] +name=Automotive-Sig \$releasever +baseurl=https://autosd.sig.centos.org/AutoSD-9/nightly/repos/AutoSD/compose/AutoSD/\$basearch/os +enabled=1 +gpgcheck=0 +EOF +} + +info_message "Starting setup" +info_message "==============================" +if [ "$EUID" -ne 0 ] +then + echo -e "[${RED} ERROR ${CLR}] Please run as root this script. It requires to set limits inside a container which is not allowed by root." + exit +fi + +info_message "Cleaning any previous e2e files" +info_message "==============================" +cleanup + +echo +info_message "Preparing QM environment" +info_message "==============================" + +# Creates the QM env on VM +if [ -n "${SET_QM_PART}" ]; then + create_qm_disks +fi +install_qm_rpms +setup_qm_services + +info_message "${GRN}QM environment${CLR}" +info_message "==============================" + +# Capture the end time +END_TIME=$(date +%s) + +# Calculate the duration in seconds +DURATION=$((END_TIME - START_TIME)) + +# Calculate minutes and seconds +DAYS=$((DURATION / 86400)) +HOURS=$(( (DURATION % 86400) / 3600 )) +MINUTES=$(( (DURATION % 3600) / 60 )) +SECONDS=$((DURATION % 60)) + +info_message "${GRN}Running time for this script${CLR}" +info_message "\t- ${DAYS} days, ${HOURS} hours, ${MINUTES} minutes and ${SECONDS} seconds" +info_message "All set!" +info_message "==============================" diff --git a/demos/devconf-2024/tools/FFI/.gitignore b/demos/devconf-2024/tools/FFI/.gitignore new file mode 100644 index 00000000..e660fd93 --- /dev/null +++ b/demos/devconf-2024/tools/FFI/.gitignore @@ -0,0 +1 @@ +bin/ diff --git a/demos/devconf-2024/tools/FFI/Makefile b/demos/devconf-2024/tools/FFI/Makefile new file mode 100644 index 00000000..b6ac69e1 --- /dev/null +++ b/demos/devconf-2024/tools/FFI/Makefile @@ -0,0 +1,38 @@ +CC = gcc +CFLAGS = -Wall -g + +VPATH=disk/QM:memory:deny_set_scheduler:deny_sched_setattr +QM_BIN=./bin/QM +ASIL_BIN=./bin/ASIL + +all: cratetestqm cratetestasil + +cratetestqm: createqmbin file-allocate 90_percent_memory_eat test_sched_setscheduler execute_sched_setattr + +cratetestasil: createasilbin 20_percent_memory_eat + +createqmbin: + @mkdir -p $(QM_BIN) + +createasilbin: + @mkdir -p $(ASIL_BIN) + +20_percent_memory_eat: memory_eat.c + $(CC) $(CFLAGS) -DMEM_PERCENT=0.2 -o $(ASIL_BIN)/$@ $< + +file-allocate: file-allocate.c + $(CC) $(CFLAGS) -o $(QM_BIN)/$@ $< + +90_percent_memory_eat: memory_eat.c + $(CC) $(CFLAGS) -DMEM_PERCENT=0.9 -o $(QM_BIN)/$@ $< + +test_sched_setscheduler: execute_set_scheduler.c + $(CC) $(CFLAGS) -o $(QM_BIN)/$@ $< + +execute_sched_setattr: execute_sched_setattr.c + $(CC) $(CFLAGS) -o $(QM_BIN)/$@ $< + +clean: + rm -rf $(QM_BIN) $(ASIL_QM) + +.PHONY: all clean createtestqm createtestasil diff --git a/demos/devconf-2024/tools/FFI/README.md b/demos/devconf-2024/tools/FFI/README.md new file mode 100644 index 00000000..fa09c058 --- /dev/null +++ b/demos/devconf-2024/tools/FFI/README.md @@ -0,0 +1,27 @@ +# Free-From-Interference + +Free-From-Interference (FFI) means applications within the QM environment do not interfere with applications in the ASIL environment. + +## Memory + +QM environment will allocates 90% or greater of memory of the system. +Launch application in the ASIL environment that requires > 10% of memory. +Make sure swap is turned off. +Make sure OOM Killer kills QM applications. + +Example: + +- `memory/ASIL/20_percent_cpu_eat.c` +- `memory/QM/90_percent_cpu_eat.c` + +## Disk + +Try to allocate as maximum possible disk space. + +## Sysctl + +Running as nested container inside QM attempt to change settings in the host level (ASIL). + +## deny_set_scheduler + +A tool to test if `set_scheduler()` is denied inside QM partition diff --git a/demos/devconf-2024/tools/FFI/deny_sched_setattr/README.md b/demos/devconf-2024/tools/FFI/deny_sched_setattr/README.md new file mode 100644 index 00000000..4b4bcd77 --- /dev/null +++ b/demos/devconf-2024/tools/FFI/deny_sched_setattr/README.md @@ -0,0 +1,27 @@ +# The `execute_sched_setattr` command + +## What is `execute_sched_setattr` ? + +A test tool to validate if `SCHED_DEADLINE` can be set via `sched_setattr()` syscall. + +## Why? + +QM environment should not allow `SCHED_DEADLINE` be set via `sched_setattr()` syscall +and must validated via FFI tests. + +## How to deny is made? + +During the QM service startup it passes arguments to Podman. One of these arguments is `seccomp=/usr/share/qm/seccomp.json` which contains rules that deny the `sched_setattr()`. + +## How to test? + +```console +host> gcc -o execute_sched_setattr execute_sched_setattr.c -Wall # build the bin +host> cp execute_sched_setattr /usr/lib/qm/rootfs/root/ # copy the bin to QM partition + +# podman exec -it qm bash # Execute the test, it must fail in recent versions of QM +bash-5.1# cd /root && ./execute_sched_setattr +Current Scheduling Policy: SCHED_OTHER +Current Priority: 0 +sched_setattr failed: Operation not permitted +``` diff --git a/demos/devconf-2024/tools/FFI/deny_sched_setattr/execute_sched_setattr.c b/demos/devconf-2024/tools/FFI/deny_sched_setattr/execute_sched_setattr.c new file mode 100644 index 00000000..5bf9a52d --- /dev/null +++ b/demos/devconf-2024/tools/FFI/deny_sched_setattr/execute_sched_setattr.c @@ -0,0 +1,91 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +struct sched_attr { + __u32 size; + + __u32 sched_policy; + __u64 sched_flags; + + // SCHED_NORMAL, SCHED_BATCH + __s32 sched_nice; + + // SCHED_FIFO, SCHED_RR + __u32 sched_priority; + + // SCHED_DEADLINE + __u64 sched_runtime; + __u64 sched_deadline; + __u64 sched_period; +}; + +int sched_setattr(pid_t pid, const struct sched_attr *attr, unsigned int flags) { + return syscall(SYS_sched_setattr, pid, attr, flags); +} + +// Function to get the scheduling policy and priority of the current process +void get_sched() { + int policy; + struct sched_param param; + + // Get current scheduling policy + policy = sched_getscheduler(0); + if (policy == -1) { + perror("sched_getscheduler"); + exit(EXIT_FAILURE); + } + + // Get the current priority + if (sched_getparam(0, ¶m) == -1) { + perror("sched_getparam"); + exit(EXIT_FAILURE); + } + + printf("Current Scheduling Policy: "); + switch (policy) { + case SCHED_OTHER: + printf("SCHED_OTHER\n"); + break; + case SCHED_FIFO: + printf("SCHED_FIFO\n"); + break; + case SCHED_RR: + printf("SCHED_RR\n"); + break; + case SCHED_DEADLINE: + printf("SCHED_DEADLINE\n"); + break; + default: + printf("Unknown\n"); + } + + printf("Current Priority: %d\n", param.sched_priority); +} + +int main() { + struct sched_attr attr; + memset(&attr, 0, sizeof(attr)); // initialization + attr.size = sizeof(attr); + attr.sched_policy = SCHED_DEADLINE; + attr.sched_runtime = 10 * 1000 * 1000; // 10 ms + attr.sched_deadline = 20 * 1000 * 1000; // 20 ms + attr.sched_period = 30 * 1000 * 1000; // 30 ms + + get_sched(); + + if (sched_setattr(0, &attr, 0) == -1) { + perror("sched_setattr failed"); + return 1; + } + + // Now the thread should be running under SCHED_DEADLINE + get_sched(); + + return 0; +} diff --git a/demos/devconf-2024/tools/FFI/deny_set_scheduler/README.md b/demos/devconf-2024/tools/FFI/deny_set_scheduler/README.md new file mode 100644 index 00000000..30c5a6cf --- /dev/null +++ b/demos/devconf-2024/tools/FFI/deny_set_scheduler/README.md @@ -0,0 +1,22 @@ +# What is `execute_set_scheduler`? + +A test tool to validate if `set_scheduler()` syscall can be executed. + +## Why? + +QM environment deny `set_scheduler()` syscall for safety and must be validated via FFI tests. + +## How to deny is made? + +During the QM service startup it passes arguments to Podman. One of these arguments is `seccomp=/usr/share/qm/seccomp.json` which contains rules that deny the `set_scheduler()`. + +## How to test? + +```console +host> gcc -o execute_set_scheduler execute_set_scheduler.c -Wall # build the bin +host> cp execute_set_scheduler /usr/lib/qm/rootfs/root/ # copy the bin to QM partition + +# podman exec -it qm bash # Execute the test, it must fail in recent versions of QM +bash-5.1# cd /root && ./test_sched_setscheduler +Failed to set scheduler: Operation not permitted +``` diff --git a/demos/devconf-2024/tools/FFI/deny_set_scheduler/execute_set_scheduler.c b/demos/devconf-2024/tools/FFI/deny_set_scheduler/execute_set_scheduler.c new file mode 100644 index 00000000..fac98b6a --- /dev/null +++ b/demos/devconf-2024/tools/FFI/deny_set_scheduler/execute_set_scheduler.c @@ -0,0 +1,28 @@ +#include +#include +#include +#include +#include +#include + +int main() { + int pid = getpid(); + int policy = SCHED_FIFO; // Desired scheduling policy + struct sched_param param; + + // Assign the maximum priority for the SCHED_FIFO policy + param.sched_priority = sched_get_priority_max(policy); + if (param.sched_priority == -1) { + fprintf(stderr, "Failed to get max priority for SCHED_FIFO: %s\n", strerror(errno)); + return EXIT_FAILURE; + } + + // Attempt to set the scheduling policy and priority + if (sched_setscheduler(pid, policy, ¶m) == -1) { + fprintf(stderr, "Failed to set scheduler: %s\n", strerror(errno)); + return EXIT_FAILURE; + } + + printf("Scheduler set to SCHED_FIFO with priority %d\n", param.sched_priority); + return EXIT_SUCCESS; +} diff --git a/demos/devconf-2024/tools/FFI/disk/QM/file-allocate.c b/demos/devconf-2024/tools/FFI/disk/QM/file-allocate.c new file mode 100644 index 00000000..48799d36 --- /dev/null +++ b/demos/devconf-2024/tools/FFI/disk/QM/file-allocate.c @@ -0,0 +1,43 @@ +#include +#include + +int main() { + const size_t chunkSize = 500L * 1024 * 1024; // 500 MB + const char *filePath = "/foobar.file"; // Change this path as needed + char *data = malloc(chunkSize); + FILE *fp; + int count = 0; + + if (data == NULL) { + perror("Failed to allocate memory"); + return 1; + } + + // Fill the data array with some dummy content + for (size_t i = 0; i < chunkSize; i++) { + data[i] = '+'; + } + + fp = fopen(filePath, "w"); + if (fp == NULL) { + perror("Failed to open file"); + free(data); + return 1; + } + + while (1) { + size_t written = fwrite(data, 1, chunkSize, fp); + if (written < chunkSize) { + printf("Failed to write after %d x 500 MB\n", count); + break; + } else { + count++; + printf("Written %d x 500 MB\n", count); + } + fflush(fp); // Make sure data is flushed to disk + } + + fclose(fp); + free(data); + return 0; +} diff --git a/demos/devconf-2024/tools/FFI/memory/memory_eat.c b/demos/devconf-2024/tools/FFI/memory/memory_eat.c new file mode 100644 index 00000000..0d2ffd3d --- /dev/null +++ b/demos/devconf-2024/tools/FFI/memory/memory_eat.c @@ -0,0 +1,49 @@ +#include +#include +#include +#include +#include + +#ifndef MEM_PERCENT +#define MEM_PERCENT 0.5 +#endif + +int main() { + struct sysinfo si; + if (sysinfo(&si) != 0) { + perror("sysinfo"); + return EXIT_FAILURE; + } + + unsigned long total_memory = si.totalram * si.mem_unit; + unsigned long memory_to_allocate = MEM_PERCENT * total_memory; + + char *buffer = malloc(memory_to_allocate); + + if (buffer == NULL) { + perror("malloc"); + return EXIT_FAILURE; + } + + // Filling the memory with zeros to ensure it's actually allocated + memset(buffer, 0, memory_to_allocate); + printf("Allocated %lu bytes out of %lu total bytes.\n", memory_to_allocate, total_memory); + + // Writing data to the allocated memory + const char *message = "Test, Test!"; + if (memory_to_allocate > strlen(message)) { + memcpy(buffer, message, strlen(message)); + // Reading and printing the written data + printf("Data in buffer: %s\n", buffer); + } else { + printf("Not enough space to write message.\n"); + } + + // Keeping the program running to hold onto the memory + while (1) { + sleep(1); + } + + free(buffer); + return EXIT_SUCCESS; +} diff --git a/demos/devconf-2024/tools/FFI/module/README.md b/demos/devconf-2024/tools/FFI/module/README.md new file mode 100644 index 00000000..006a9878 --- /dev/null +++ b/demos/devconf-2024/tools/FFI/module/README.md @@ -0,0 +1,21 @@ +# The `modprobe_module` command + +## What is `modprobe_module`? + +`modprobe_module` is a simple script that validates it won't be possible to access `/lib/modules` to load any module via modprobe inside the QM partition. + +## How to use it? + +It must be executed inside the QM partition to validate it won't be possible to modprobe any module under `/lib/modules`. + +Example: + +```console +my-host# podman exec -it qm bash + +bash-5.1# ./modprobe_module +modprobe: FATAL: Module ext4 not found in directory /lib/modules/5.14.0-447.400.el9iv.x86_64 +ls: cannot access '/lib/modules/5.14.0-447.400.el9iv.x86_64': No such file or directory +done + +``` diff --git a/demos/devconf-2024/tools/FFI/module/modprobe_module b/demos/devconf-2024/tools/FFI/module/modprobe_module new file mode 100644 index 00000000..bbcfd231 --- /dev/null +++ b/demos/devconf-2024/tools/FFI/module/modprobe_module @@ -0,0 +1,19 @@ +#!/bin/bash + +if [ ! -d /lib/modules ]; then + echo "FATAL: /lib/modules: No such file or directory" + exit 1 +fi + +if [ -n "$(ls -A /lib/modules)" ]; then + echo "This folder should be empty, any modules cannot load via modprobe" + exit 1 +fi + +# Modprobe module ext4 +modprobe ext4 + +# Access module file +ls $(modprobe ext4 2>&1 >/dev/null | cut -d ' ' -f 9) + +echo "done" diff --git a/demos/devconf-2024/tools/FFI/sysctl/README.md b/demos/devconf-2024/tools/FFI/sysctl/README.md new file mode 100644 index 00000000..5ab4a329 --- /dev/null +++ b/demos/devconf-2024/tools/FFI/sysctl/README.md @@ -0,0 +1,26 @@ +# The `setsysctl` command + +## What is setsysctl? + +setsysctl is simple script that uses sysctl tool to try to change several parameters in the host OS. + +## How to use it? + +It must be executed inside QM environment as a container (nested) to make sure attemps to change OS level are denied. + +Example: + +```console +my-host# podman exec -it qm bash + +bash-5.1# podman run -it fedora bash + +[root@d0c4f92289b0 ~]# ./setsysctl +Setting sysctl parameters... +sysctl: permission denied on key "net.ipv4.ip_forward" +sysctl: permission denied on key "net.ipv4.conf.all.rp_filter" +sysctl: permission denied on key "net.ipv4.tcp_max_syn_backlog" +sysctl: permission denied on key "vm.swappiness" +sysctl: permission denied on key "vm.overcommit_memory" +done +``` diff --git a/demos/devconf-2024/tools/FFI/sysctl/setsysctl b/demos/devconf-2024/tools/FFI/sysctl/setsysctl new file mode 100755 index 00000000..41c389c5 --- /dev/null +++ b/demos/devconf-2024/tools/FFI/sysctl/setsysctl @@ -0,0 +1,35 @@ +#!/bin/bash + +if ! command -v sysctl &>/dev/null; then + echo "sysctl tool is not installed. Installing..." + sudo dnf install -y procps-ng +fi + +echo "Setting sysctl parameters..." + +# Generate a random number within a given range +generate_random_number() { + local min=$1 + local max=$2 + echo $((RANDOM % (max - min + 1) + min)) +} + +# Network tuning parameters +sysctl -w net.ipv4.ip_forward="$(generate_random_number 0 1)" +sysctl -w net.ipv4.conf.all.rp_filter="$(generate_random_number 0 2)" +sysctl -w net.ipv4.tcp_max_syn_backlog="$(generate_random_number 128 1024)" + +# Buffer Sizes +if [ -f "/proc/sys/net/core/rmem_max" ]; then + sysctl -w net.core.rmem_max="$(generate_random_number 4096 16777216)" +fi + +if [ -f "/proc/sys/net/core/wmem_max" ]; then + sysctl -w net.core.wmem_max="$(generate_random_number 4096 16777216)" +fi + +# Memory management settings +sysctl -w vm.swappiness="$(generate_random_number 0 100)" +sysctl -w vm.overcommit_memory="$(generate_random_number 0 2)" + +echo "done" diff --git a/demos/devconf-2024/tools/README.md b/demos/devconf-2024/tools/README.md new file mode 100644 index 00000000..1232431c --- /dev/null +++ b/demos/devconf-2024/tools/README.md @@ -0,0 +1,5 @@ +The `remove-containers` script will remove all containers (and images) which name start with node* or control from the output of podman ps. Useful to cleanup environment before executing tests and measures. + +### Freedom From Interference + +Under FFI binaries build in different [automotive-repo](https://gitlab.com/CentOS/automotive/container-images/-/tree/main/images/ffi-tools) diff --git a/demos/devconf-2024/tools/remove-containers b/demos/devconf-2024/tools/remove-containers new file mode 100755 index 00000000..912a160f --- /dev/null +++ b/demos/devconf-2024/tools/remove-containers @@ -0,0 +1,15 @@ +#!/bin/bash + +output=$(podman ps -a --format "{{.Names}}") +IFS=$'\n' # Set the internal field separator to newline + +# DO NOT use double quotes here +for node in ${output}; do + if [[ "${node}" == "control" ]] || [[ "${node}" =~ ^node.* ]]; then + echo "Removing container ${node}..." + podman rm --storage --force "${node}" &> /dev/null + podman rm "${node}" --force 1> /dev/null + fi +done + +podman network rm podmanDualStack -f