[rhcos-4.16] overlay.d: add 07fix-selinux-labels overlay

manifests/shared-workarounds.yaml

@@ -1,2 +1,4 @@
 # This manifest is a list of shared workarounds that are needed in both Fedora CoreOS
 # and downstreams (i.e. Red Hat CoreOS).
+ostree-layers:
+  - overlay/07fix-selinux-labels

overlay.d/07fix-selinux-labels/usr/lib/systemd/system-preset/40-fix-selinux-labels.preset

@@ -0,0 +1,8 @@
+# Fix incorrect SELinux labels in /boot and /sysroot
+# - https://github.com/coreos/fedora-coreos-tracker/issues/1772
+# - https://github.com/coreos/fedora-coreos-tracker/issues/1771
+# We need this for both FCOS and RHCOS and it needs to live for
+# some time (not just a single FCOS barrier release) so that we
+# can ensure RHCOS 4.16 aleph nodes and some early 4.17 aleph
+# nodes have been fixed.
+enable coreos-fix-selinux-labels.service

overlay.d/07fix-selinux-labels/usr/lib/systemd/system/coreos-fix-selinux-labels.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=Fix mislabeled or unlabeled SELinux contexts on files
+Documentation=https://github.com/coreos/fedora-coreos-tracker/issues/1771
+Documentation=https://github.com/coreos/fedora-coreos-tracker/issues/1772
+ConditionPathExists=!/var/lib/coreos-fix-selinux-labels.stamp
+
+[Service]
+Type=oneshot
+# Don't run this more than once, even if it fails
+ExecStartPre=/bin/touch /var/lib/coreos-fix-selinux-labels.stamp
+ExecStart=/usr/libexec/coreos-fix-selinux-labels
+RemainAfterExit=yes
+MountFlags=slave
+
+[Install]
+WantedBy=multi-user.target

overlay.d/07fix-selinux-labels/usr/libexec/coreos-fix-selinux-labels

@@ -0,0 +1,165 @@
+#!/usr/bin/bash
+
+# Script to help fix selinux labels on systems that were created with
+# OSBuild before https://github.com/coreos/coreos-assembler/commit/d3302e0fc9bedec2d4e935e3528eb5abd44e7ae8
+# was put in place to ensure images didn't get created with unlabeled
+# or mislabeled files. See also
+# - https://github.com/coreos/fedora-coreos-tracker/issues/1771
+# - https://github.com/coreos/fedora-coreos-tracker/issues/1772
+#
+# Also handle /boot/.root_uuid and /boot/grub2/bootuuid.cfg created
+# by rdcore for some time without labels.
+# - https://github.com/coreos/fedora-coreos-tracker/issues/1770
+# - https://github.com/coreos/fedora-coreos-config/pull/3155
+
+set -eu -o pipefail
+
+print_header() {
+    echo '------------------------------------'
+    echo "$1"
+    echo
+}
+
+get_context() {
+    path=$1
+    getfattr -n security.selinux --absolute-name --only-values "${path}" | \
+        tr -d '\0' # Trim the null byte from the ouptut to prevent bash warning
+}
+
+path_unlabeled() {
+    test -e "$1" || return 1 # no exist so not unlabeled
+    if [ "$(get_context "$1")" == "system_u:object_r:unlabeled_t:s0" ]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+any_unlabeled() {
+    # shellcheck disable=SC2068
+    for file in $@; do
+        path_unlabeled "${file}" && return 0
+    done
+    return 1 # none were unlabeled
+}
+
+# Check a few known paths. If /sysroot is unlabeled then we need to
+# clean up the mess that OSBuild left behind #1771,#1772. If /boot/.root_uuid
+# or /boot/grub2/bootuuid.cfg are unlabeled we need to fix those two #1770.
+if ! any_unlabeled '/sysroot' '/boot/.root_uuid' '/boot/grub2/bootuuid.cfg'; then
+    echo "This CoreOS installation is properly labeled. Exiting"
+    exit 0
+fi
+
+print_header "Remounting filesystems read/write"
+# Note we don't need to remount them read-only later
+# because we are running with MountFlags=slave so
+# changes here won't propagate to the rest of the system
+mount -v -o remount,rw /boot
+mount -v -o remount,rw /sysroot
+
+# Fix the few ones we know about. Some of these are from #1770
+# and some from #1772, but it's easier to just combine the code.
+print_header "Fixing label for files on the /boot filesystem"
+for file in '.root_uuid' 'grub2/bootuuid.cfg' 'lost+found'; do
+    if path_unlabeled "/boot/${file}"; then
+        context=$(matchpathcon -n "/boot/${file}")
+        echo "Changing context of /boot/${file} to ${context}"
+        /usr/bin/chcon -h -v "${context}" "/boot/${file}"
+    fi
+done
+# Also handle coreos/platforms.json, which could have the wrong label
+if [ -e "/boot/coreos/platforms.json" ]; then
+    restorecon -v "/boot/coreos/platforms.json"
+fi
+
+if ! path_unlabeled "/sysroot"; then
+    # We don't need to go further with the other fixes since
+    # this system doesn't appear to be affected by #1771,#1772.
+    echo "coreos-fix-selinux-labels finished successfully" > /var/lib/coreos-fix-selinux-labels.stamp
+    exit 0
+fi
+
+print_header "Mounting boot partition separately to check shadowed /boot/efi"
+boot_mount_point=$(mktemp --directory)
+mount -v /dev/disk/by-label/boot "$boot_mount_point"
+if path_unlabeled "${boot_mount_point}/efi"; then
+    echo "Fixing label on shadowed /boot/efi"
+    context=$(matchpathcon -n "/boot/efi")
+    echo "Changing context of /boot/efi to ${context}"
+    /usr/bin/chcon -h -v "${context}" "${boot_mount_point}/efi"
+fi
+umount -v "$boot_mount_point"
+rmdir "$boot_mount_point"
+
+# The underlying /boot directory on the root filesystem can be wrong
+print_header "Checking shadowed /boot"
+if path_unlabeled "/sysroot/boot"; then
+    echo "Fixing the label for the /boot mount point on the root filesystem"
+    context=$(matchpathcon -n "/boot/")
+    echo "Changing context of /sysroot/boot to ${context}"
+    /usr/bin/chcon -h -v "${context}" "/sysroot/boot"
+fi
+
+# Fix unlabeled files. The find commands are hand crafted to try
+# to catch all unlabeled files, but not touch any objects in the
+# ostree repo and also not traverse too deep in the filesystem,
+# which could take more time than we'd like.
+#
+#  - /ostree/repo/refs/ to capture the container/blob/ files
+#  - /ostree/boot* to capture boot.x and bootx.x files
+#  - /ostree/repo/{.lock,config} - two known offenders
+#  - .aleph-version.json, .coreos-aleph-version.json - two more
+#  - -type l -or -type d  - all directories and symlinks in the repo
+#  - -type f -regex '.*\.\(commitmeta\|commit\|dirmeta\|dirtree\|origin\)$'
+#       - all .commitmeta, .commit, .dirmeta, .dirtree, .origin
+#         files in the repo and no other files (objects)
+#
+# Note that we explicitly prune /sysroot/ostree/deploy/*/var so we
+# don't consider anything under that path for our operation. Note
+# also some of these are left unquoted to allow for shell expansion.
+#
+context=$(matchpathcon -n "/")
+tmpfile=$(mktemp)
+print_header "Changing context of unlabeled files to ${context}"
+(
+    find "/sysroot/ostree/repo/refs"           \
+         "/sysroot/.aleph-version.json"        \
+         "/sysroot/.coreos-aleph-version.json" \
+         /sysroot/ostree/repo/{.lock,config}   \
+         /sysroot/ostree/boot*                 \
+             -context '*:unlabeled_t:*' -print0;
+    find "/sysroot/" -maxdepth 7 -path /sysroot/ostree/deploy/*/var -prune -o                \
+        \(                                                                                   \
+            -context '*:unlabeled_t:*'                                                       \
+            \(                                                                               \
+                -type l -or -type d -or                                                      \
+                \( -type f -regex '.*\.\(commitmeta\|commit\|dirmeta\|dirtree\|origin\)$' \) \
+            \)                                                                               \
+            -print0                                                                          \
+        \)
+) | xargs --null -I{} chcon -v -h "${context}" {} > "${tmpfile}"
+# Print something here for the journal, but not the full list of files
+# because that would be a lot. We'll dump those in the stamp file later.
+echo "Relabeled $(wc -l < "${tmpfile}") files to ${context}"
+
+# Update the stamp file with a record of what was done up until this point
+journalctl -b0 -u coreos-fix-selinux-labels.service >> /var/lib/coreos-fix-selinux-labels.stamp
+print_header "The following are the unlabeled files that were fixed" >> /var/lib/coreos-fix-selinux-labels.stamp
+cat "${tmpfile}" >> /var/lib/coreos-fix-selinux-labels.stamp
+rm -f "${tmpfile}"
+timestamp=$(date +%s)
+
+print_header "Checking the repository for consistency"
+if ! ostree fsck; then
+    echo "OSTree fsck found corruption. Please reprovision if you can or"  1>&2
+    echo "ask for help at https://discussion.fedoraproject.org/tag/coreos" 1>&2
+    echo "coreos-fix-selinux-labels finished with failure" > /var/lib/coreos-fix-selinux-labels.stamp
+    exit 1
+fi
+
+# Capture the final bits in the stamp file
+journalctl --since="@${timestamp}" -u coreos-fix-selinux-labels.service >> /var/lib/coreos-fix-selinux-labels.stamp
+
+# This will go to both the journal and the stamp file
+echo "coreos-fix-selinux-labels finished successfully" | tee -a /var/lib/coreos-fix-selinux-labels.stamp

overlay.d/README.md

@@ -10,6 +10,17 @@ This overlay matches `fedora-coreos-base.yaml`; core Ignition+ostree bits.
 
 This overlay is shared with RHCOS/SCOS 9.
 
+07fix-selinux-labels
+--------------------
+
+Fix incorrect SELinux labels in /boot and /sysroot
+- https://github.com/coreos/fedora-coreos-tracker/issues/1772
+- https://github.com/coreos/fedora-coreos-tracker/issues/1771
+We need this for both FCOS and RHCOS and it needs to live for
+some time (not just a single FCOS barrier release) so that we
+can ensure RHCOS 4.16 aleph nodes and some early 4.17 aleph
+nodes have been fixed. Remove it in the 4.19 cycle.
+
 08nouveau
 ---------
 

tests/kola/selinux/file-context-policy-match

@@ -0,0 +1,50 @@
+#!/bin/bash
+## kola:
+##   exclusive: false
+##   tags: "platform-independent"
+##   description: Verify there are no unlabeled or mislabeled files on the system.
+
+# See https://github.com/coreos/fedora-coreos-tracker/issues/1772
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+# Add the `-not -regex '.*.0/var/mnt'` here to have an exception for
+# the /sysroot/ostree/deploy/*/deploy/*.0/var/mnt directory existing
+# and being unlabeled. This is created by systemd pre-v254 during
+# pivoting (and so before the policy is loaded) as a temporary
+# mountpoint. We can remove it once we support el10+ only.
+unlabeled="$(find /sysroot -context '*unlabeled_t*' -not -regex '.*.0/var/mnt' -print0 | xargs --null -I{} ls -ldZ '{}')"
+if [ -n "${unlabeled}" ]; then
+    fatal "Some unlabeled files were found"
+fi
+
+mislabeled="$(restorecon -vnr /var/ /etc/ /usr/ /boot/)"
+if [ -n "${mislabeled}" ]; then
+    # Exceptions for files that could be wrong
+    # On RHCOS
+    #   - Would relabel /var/opt/cni from system_u:object_r:usr_t:s0 to system_u:object_r:var_t:s0
+    #     - https://github.com/openshift/os/issues/1624
+    #   - Would relabel /etc/iscsi/initiatorname.iscsi from system_u:object_r:etc_runtime_t:s0 to system_u:object_r:etc_t:s0
+    #     - Fixed by https://github.com/openshift/os/pull/1622
+    #     - Remove when the oldest supported RHCOS release is 4.18 or newer
+    declare -A exceptions=(
+        ['/var/opt/cni']=1
+        ['/etc/iscsi/initiatorname.iscsi']=1
+    )
+    paths="$(echo "${mislabeled}" | grep "Would relabel" | cut -d ' ' -f 3)"
+    found=""
+    while read -r path; do
+        if [[ "${exceptions[$path]:-noexception}" == 'noexception' ]]; then
+            echo "Unexpected mislabeled file found: ${path}"
+            found="1"
+        fi
+    done <<< "${paths}"
+    if [ "${found}" == "1" ];then
+        fatal "Some unexpected mislabeled files were found."
+    fi
+fi
+
+ok "No unlabeled or mislabeled files found!"

tests/kola/upgrade/extended/test.sh

@@ -138,6 +138,49 @@ EOF
     need_zincati_restart='true'
 }
 
+selinux-sanity-check() {
+    # First make sure the migrations/fix script has finished (if it is going
+    # to run) before doing the checks
+    systemd-run --wait --property=After=coreos-fix-selinux-labels.service \
+        echo "Waited for coreos-fix-selinux-labels.service to finish"
+    # Verify SELinux labels are sane. Migration scripts should have cleaned
+    # up https://github.com/coreos/fedora-coreos-tracker/issues/1772
+    unlabeled="$(find /sysroot -context '*unlabeled_t*' -print0 | xargs --null -I{} ls -ldZ '{}')"
+    if [ -n "${unlabeled}" ]; then
+        fatal "Some unlabeled files were found"
+    fi
+    mislabeled="$(restorecon -vnr /var/ /etc/ /usr/ /boot/)"
+    if [ -n "${mislabeled}" ]; then
+        # Exceptions for files that could be wrong (sometimes upgrades are messy)
+        #   Would relabel /var/lib/cni from system_u:object_r:var_lib_t:s0 to system_u:object_r:container_var_lib_t:s0
+        #   Would relabel /etc/selinux/targeted/semanage.read.LOCK from system_u:object_r:semanage_trans_lock_t:s0 to system_u:object_r:selinux_config_t:s0
+        #   Would relabel /etc/selinux/targeted/semanage.trans.LOCK from system_u:object_r:semanage_trans_lock_t:s0 to system_u:object_r:selinux_config_t:s0
+        #   Would relabel /etc/systemd/journald.conf.d from system_u:object_r:etc_t:s0 to system_u:object_r:systemd_conf_t:s0
+        #   Would relabel /etc/systemd/journald.conf.d/forward-to-console.conf from system_u:object_r:etc_t:s0 to system_u:object_r:systemd_conf_t:s0
+        #   Would relabel /boot/lost+found from system_u:object_r:unlabeled_t:s0 to system_u:object_r:lost_found_t:s0' ']'
+        declare -A exceptions=(
+           ['/var/lib/cni']=1
+           ['/etc/selinux/targeted/semanage.read.LOCK']=1
+           ['/etc/selinux/targeted/semanage.trans.LOCK']=1
+           ['/etc/systemd/journald.conf.d']=1
+           ['/etc/systemd/journald.conf.d/forward-to-console.conf']=1
+           ['/boot/lost+found']=1
+        )
+        paths="$(echo "${mislabeled}" | grep "Would relabel" | cut -d ' ' -f 3)"
+        found=""
+        while read -r path; do
+            if [[ "${exceptions[$path]:-noexception}" == 'noexception' ]]; then
+                echo "Unexpected mislabeled file found: ${path}"
+                found="1"
+            fi
+        done <<< "${paths}"
+        if [ "${found}" == "1" ];then
+            fatal "Some unexpected mislabeled files were found."
+        fi
+    fi
+    ok "Selinux sanity checks passed"
+}
+
 ok "Reached version: $version"
 
 # Are we all the way at the desired target version?
@@ -146,6 +189,8 @@ if vereq $version $target_version; then
     ok "Fully upgraded to $target_version"
     # log bootupctl information for inspection where available
     [ -f /usr/bin/bootupctl ] && /usr/bin/bootupctl status
+    # One last check!
+    selinux-sanity-check
     exit 0
 fi