Add bluechi-agent flood test

tests/e2e/set-ffi-env-e2e

@@ -188,13 +188,14 @@ install_qm_rpms() {
   if [ -n "${USE_QM_COPR}" ]; then
       dnf copr enable -y "${USE_QM_COPR}" centos-stream-9-"${ARCH}"
   fi
-  dnf install -y qm bluechi-ctl bluechi podman hostname
+  dnf install -y bluechi-ctl bluechi-agent bluechi qm podman hostname
 }
 
 setup_qm_services() {
 
   info_message "Setup qm services"
   info_message "=============================="
+
   #Update setup script from QM repos, in case rpm not updates yet
   if [ -n "${QM_GH_URL}" ]; then
       curl "${QM_GH_URL}"  > /usr/share/qm/setup
@@ -238,6 +239,7 @@ then
 fi
 
 info_message "Cleaning any previous e2e files"
+info_message "=============================="
 cleanup
 
 echo
@@ -252,7 +254,7 @@ install_qm_rpms
 setup_qm_services
 
 info_message "${GRN}QM environment${CLR}"
-echo
+info_message "=============================="
 
 # Capture the end time
 END_TIME=$(date +%s)
@@ -266,9 +268,7 @@ HOURS=$(( (DURATION % 86400) / 3600 ))
 MINUTES=$(( (DURATION % 3600) / 60 ))
 SECONDS=$((DURATION % 60))
 
-echo
 info_message "${GRN}Running time for this script${CLR}"
 info_message "\t- ${DAYS} days, ${HOURS} hours, ${MINUTES} minutes and ${SECONDS} seconds"
-echo
 info_message "All set!"
-echo
+info_message "=============================="

tests/ffi/agent-flood/main.fmf

@@ -0,0 +1,10 @@
+summary: Test bluechi evel agents
+test:
+    /bin/bash ./test.sh
+duration: 10m
+tag: ffi
+#This tests should run last
+order: 70
+
+framework:
+    shell

tests/ffi/agent-flood/test.sh

@@ -0,0 +1,93 @@
+#!/bin/bash -euvx
+
+# shellcheck disable=SC1091
+
+. ../common/prepare.sh
+
+export QM_HOST_REGISTRY_DIR="/var/qm/lib/containers/registry"
+export QM_REGISTRY_DIR="/var/lib/containers/registry"
+export NUMBER_OF_NODES="${NUMBER_OF_NODES:-2}"
+WAIT_BLUECHI_AGENT_CONNECT="${WAIT_BLUECHI_AGENT_CONNECT:-5}"
+
+setup_test_containers_in_qm() {
+
+    #Prepare quadlet files for testing containers
+    for ((i=1;i<=NUMBER_OF_NODES;i++)); do
+        info_message "setup_test_containers_in_qm(): prepare quadlet files for bluechi-tester-${i}.container"
+        cat >> "/etc/qm/containers/systemd/bluechi-tester-${i}.container" <<EOF
+[Unit]
+Description=bluechi-tester-X
+After=local-fs.target
+
+[Container]
+Image=dir:/var/lib/containers/registry/tools-ffi:latest
+Exec=/root/tests/FFI/bin/bluechi-tester --url="tcp:host=localhost,port=842" \
+     --nodename=bluechi-tester-X \
+     --numbersignals=11111111 \
+     --signal="JobDone"
+Network=host
+EOF
+        sed -i -e "s/tester-X/tester-${i}/g" "/etc/qm/containers/systemd/bluechi-tester-${i}.container"
+
+        info_message "setup_test_containers_in_qm(): updating AllowedNodeNames in /etc/bluechi/controller.conf"
+        #Update controller configuration
+        sed -i -e '/^AllowedNodeNames=/ s/$/,bluechi-tester-'"${i}"'/' /etc/bluechi/controller.conf
+
+        info_message "setup_test_containers_in_qm(): bluechi-controller reload & restart"
+        #Reload bluechi-controller
+        exec_cmd "systemctl daemon-reload"
+        exec_cmd "systemctl restart bluechi-controller"
+
+    done
+
+    #Restart bluechi-agent for clean connection logs
+    exec_cmd "systemctl restart bluechi-agent"
+    sleep "${WAIT_BLUECHI_AGENT_CONNECT}"
+    if [ "$(systemctl is-active bluechi-agent)" != "active" ]; then
+        info_message "setup_test_containers_in_qm(): bluechi-agent is not active"
+        exit 1
+    fi
+}
+
+run_test_containers(){
+    for ((i=1;i<=NUMBER_OF_NODES;i++)); do
+        #Reload bluechi-testers in qm
+        info_message "run_test_containers(): bluechi-tester-${i} reload & restart"
+        exec_cmd "podman exec qm systemctl daemon-reload"
+        exec_cmd "podman exec qm systemctl restart bluechi-tester-${i}"
+    done
+}
+
+disk_cleanup
+prepare_test
+reload_config
+prepare_images
+
+#Stop QM bluechi-agent
+exec_cmd "podman exec -it qm /bin/bash -c \
+         \"systemctl stop bluechi-agent\""
+
+#Prepare quadlet files for testing containers
+setup_test_containers_in_qm
+#Run containers through systemd
+run_test_containers
+
+#Check both tests services are on
+for ((i=1;i<=NUMBER_OF_NODES;i++)); do
+    if [ "$(podman exec qm systemctl is-active bluechi-tester-"${i}")" != "active" ]; then
+        info_message "test() bluechi-tester-${i} is not active"
+        exit 1
+    fi
+done
+
+#check ASIL bluechi-agent is connected
+connection_cnt="$(grep -e Connected -e "'localrootfs'" \
+    -oc <<< "$(systemctl status -l --no-pager bluechi-agent)")"
+if [ "${connection_cnt}" -ne 1 ]; then
+    info_message "test() expects ASIL bluechi-agent connection not disturbed"
+    agent_log="$(grep "bluechi-agent\[.*\]:" <<< "$(systemctl status -l --no-pager bluechi-agent)")"
+    info_message "test() agent logs..."
+    info_message "test() ${agent_log}"
+    info_message "test() number of connections found ${connection_cnt}"
+    exit "${connection_cnt}"
+fi

tests/ffi/common/prepare.sh

@@ -17,6 +17,8 @@ prepare_test() {
    if ! grep "Volatile" "${qm_service_file}" ; then
      exec_cmd "sed -i 's|ReadOnly=true|&\nVolatileTmp=true|' ${qm_service_file}"
    fi
+   # FIXME: QM is failing to start run podman #297 on asil space
+   exec_cmd "restorecon -RFv /var/lib/containers &> /tmp/asil-restorecon"
 }
 
 disk_cleanup() {
@@ -32,3 +34,32 @@ reload_config() {
    exec_cmd "systemctl daemon-reload"
    exec_cmd "systemctl restart qm"
 }
+
+prepare_images() {
+   exec_cmd "podman pull quay.io/centos-sig-automotive/ffi-tools:latest"
+   # Copy container image registry to /var/qm/lib/containers
+   image_id=$(podman images | grep quay.io/centos-sig-automotive/ffi-tools | awk -F " " '{print $3}')
+   if [ ! -d "${QM_HOST_REGISTRY_DIR}" ]; then
+       exec_cmd "mkdir -p ${QM_HOST_REGISTRY_DIR}"
+       exec_cmd "podman push ${image_id} dir:${QM_HOST_REGISTRY_DIR}/tools-ffi:latest"
+       # Remove image to save /var space
+       exec_cmd "podman image rm ${image_id}"
+   fi
+}
+
+run_container_in_qm() {
+   local container_name
+   local tmp_image_dir
+   local run_ctr_in_qm
+   # Clean tmp image directory
+   tmp_image_dir=$(podman info | grep  CopyTmp | awk -F":" '{print $2}')
+   container_name="${1}"
+
+   exec_cmd "podman exec -it qm /bin/bash -c \
+      \"rm -rf ${tmp_image_dir}/*\""
+   run_ctr_in_qm="podman exec -it qm /bin/bash -c \
+      \"podman run -d --net host  --replace --name ${container_name} \
+      dir:${QM_REGISTRY_DIR}/tools-ffi:latest \
+      tail -f /dev/null\""
+   exec_cmd "${run_ctr_in_qm}"
+}