diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index 88e7d83340..137365e902 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -42,6 +42,18 @@ import ( mcfglistersv1 "github.com/openshift/machine-config-operator/pkg/generated/listers/machineconfiguration.openshift.io/v1" ) +// Make this an interface so that we can easily inject mock versions of this. +type osRelease interface { + IsEL() bool + IsEL9() bool + IsCoreOSVariant() bool + IsFCOS() bool + IsSCOS() bool + IsLikeTraditionalRHEL7() bool +} + +var _ osRelease = osrelease.OperatingSystem{} + // Daemon is the dispatch point for the functions of the agent on the // machine. it keeps track of connections and the current state of the update // process. @@ -50,7 +62,7 @@ type Daemon struct { name string // os the operating system the MCD is running on - os osrelease.OperatingSystem + os osRelease // mock is set if we're running as non-root, probably under unit tests mock bool diff --git a/pkg/daemon/osrelease/osrelease.go b/pkg/daemon/osrelease/osrelease.go index cef17a7321..726cd16092 100644 --- a/pkg/daemon/osrelease/osrelease.go +++ b/pkg/daemon/osrelease/osrelease.go @@ -1,6 +1,7 @@ package osrelease import ( + "fmt" "os" "path/filepath" "strings" @@ -8,6 +9,16 @@ import ( "github.com/ashcrow/osrelease" ) +// Source of the OS release information +type InfoSource string + +const ( + // From the /etc/os-release / /usr/lib/os-release files. + OSReleaseInfoSource InfoSource = "OS Release" + // From the OS image labels. + ImageLabelInfoSource InfoSource = "OS Image Label" +) + // OS Release Paths const ( EtcOSReleasePath string = "/etc/os-release" @@ -25,17 +36,57 @@ const ( // OperatingSystem is a wrapper around a subset of the os-release fields // and also tracks whether ostree is in use. type OperatingSystem struct { - // id is the ID field from the os-release + // id is the ID field from the os-release or inferred from the OS image + // label. id string - // variantID is the VARIANT_ID field from the os-release + // variantID is the VARIANT_ID field from the os-release or inferred from the + // OS image label. variantID string - // version is the VERSION, RHEL_VERSION, or VERSION_ID field from the os-release + // version is the VERSION, RHEL_VERSION, or VERSION_ID field from the + // os-release or image label. version string - // osrelease is the underlying struct from github.com/ashcrow/osrelease - osrelease osrelease.OSRelease + // values is a map of all the values we uncovered either via the + // /etc/os-release / /usr/lib/os-release files *or* the labels attached + // to an OS image. + values map[string]string + // source identifies whether this came from the OSRelease file or from image labels. + source InfoSource +} + +func newOperatingSystemFromImageLabels(imageLabels map[string]string) (OperatingSystem, error) { + if err := hasRequiredLabels(imageLabels); err != nil { + return OperatingSystem{}, err + } + + os := OperatingSystem{ + values: imageLabels, + source: ImageLabelInfoSource, + // If we've made it this far, we know we have a CoreOS variant. + variantID: coreos, + version: imageLabels["version"], + } + + // Only FCOS and SCOS set this label, which is why it's not required. + if osName, osNameOK := imageLabels["io.openshift.build.version-display-names"]; osNameOK { + return inferNonRHCOS(os, osName) + } + + // Like SCOS, RHCOS has the version number in the middle position of the OCP + // / OKD version ID (e.g., 413.92.202302081904-0, becomes 92; which is 9.2 + // though we don't care about the missing decimal here) + os.version = strings.Split(os.version, ".")[1] + + // If we've made it this far and the first character is either 8 or 9, we + // most likely have an RHCOS image. + if os.version[0:1] == "8" || os.version[0:1] == "9" { + os.id = rhcos + return os, nil + } + + return os, fmt.Errorf("unable to infer OS version from image labels: %v", imageLabels) } -func newOperatingSystem(etcPath, libPath string) (OperatingSystem, error) { +func newOperatingSystemFromOSRelease(etcPath, libPath string) (OperatingSystem, error) { ret := OperatingSystem{} or, err := osrelease.NewWithOverrides(etcPath, libPath) @@ -46,14 +97,37 @@ func newOperatingSystem(etcPath, libPath string) (OperatingSystem, error) { ret.id = or.ID ret.variantID = or.VARIANT_ID ret.version = getOSVersion(or) - ret.osrelease = or + + // Store all of the values identified by the osrelease library. + ret.values = or.ADDITIONAL_FIELDS + ret.values["NAME"] = or.NAME + ret.values["VERSION"] = or.VERSION + ret.values["ID"] = or.ID + ret.values["ID_LIKE"] = or.ID_LIKE + ret.values["VERSION_ID"] = or.VERSION_ID + ret.values["VERSION_CODENAME"] = or.VERSION_CODENAME + ret.values["PRETTY_NAME"] = or.PRETTY_NAME + ret.values["ANSI_COLOR"] = or.ANSI_COLOR + ret.values["CPE_NAME"] = or.CPE_NAME + ret.values["HOME_URL"] = or.HOME_URL + ret.values["BUG_REPORT_URL"] = or.BUG_REPORT_URL + ret.values["PRIVACY_POLICY_URL"] = or.PRIVACY_POLICY_URL + ret.values["VARIANT"] = or.VARIANT + ret.values["VARIANT_ID"] = or.VARIANT_ID + + ret.source = OSReleaseInfoSource return ret, nil } -// Returns the underlying OSRelease struct if additional parameters are needed. -func (os OperatingSystem) OSRelease() osrelease.OSRelease { - return os.osrelease +// Returns the source of where this info came from. +func (os OperatingSystem) Source() InfoSource { + return os.source +} + +// Returns the values map if cdditional ontext is needed. +func (os OperatingSystem) Values() map[string]string { + return os.values } // IsEL is true if the OS is an Enterprise Linux variant, @@ -64,7 +138,7 @@ func (os OperatingSystem) IsEL() bool { // IsEL9 is true if the OS is RHCOS 9 or SCOS 9 func (os OperatingSystem) IsEL9() bool { - return os.IsEL() && strings.HasPrefix(os.version, "9.") || os.version == "9" + return os.IsEL() && strings.HasPrefix(os.version, "9") || os.version == "9" } // IsFCOS is true if the OS is Fedora CoreOS @@ -102,7 +176,7 @@ func (os OperatingSystem) ToPrometheusLabel() string { // GetHostRunningOS reads os-release to generate the OperatingSystem data. func GetHostRunningOS() (OperatingSystem, error) { - return newOperatingSystem(EtcOSReleasePath, LibOSReleasePath) + return newOperatingSystemFromOSRelease(EtcOSReleasePath, LibOSReleasePath) } // Generates the OperatingSystem data from strings which contain the desired @@ -126,7 +200,62 @@ func LoadOSRelease(etcOSReleaseContent, libOSReleaseContent string) (OperatingSy return OperatingSystem{}, err } - return newOperatingSystem(etcOSReleasePath, libOSReleasePath) + return newOperatingSystemFromOSRelease(etcOSReleasePath, libOSReleasePath) +} + +// Infers the OS release version given the image labels from a given OS image. +func InferFromOSImageLabels(imageLabels map[string]string) (OperatingSystem, error) { + return newOperatingSystemFromImageLabels(imageLabels) +} + +// Determines if an OS image has the labels that are required to infer what OS +// it contains. +func hasRequiredLabels(imageLabels map[string]string) error { + requiredLabels := []string{ + "coreos-assembler.image-input-checksum", + "coreos-assembler.image-config-checksum", + "org.opencontainers.image.revision", + "org.opencontainers.image.source", + "version", + } + + for _, reqLabel := range requiredLabels { + if _, ok := imageLabels[reqLabel]; !ok { + return fmt.Errorf("labels %v missing required key %q", imageLabels, reqLabel) + } + } + + return nil +} + +// Infers that a given oeprating system is either FCOS or SCOS. +func inferNonRHCOS(os OperatingSystem, osName string) (OperatingSystem, error) { + osName = strings.ReplaceAll(osName, "machine-os=", "") + + switch osName { + case "CentOS Stream CoreOS": + os.id = scos + // Grab the middle value from the version number (e.g., + // 413.9.202302130811-0 becomes 9) + os.version = strings.Split(os.version, ".")[1] + case "Fedora CoreOS": + // FCOS doesn't have the OCP / OKD version number encoded in it (e.g., + // 37.20230211.20.0) so we don't need to mutate it or inspect it. + os.id = fedora + default: + // Catch-all if we have an unknown OS name. + return os, fmt.Errorf("unknown OS %q", osName) + } + + // Currently, SCOS is at major version 9. This will probably change in the + // distant future. Additionally, this provides a guard in the event that + // the version number schema changes. + if os.id == scos && os.version != "9" { + return os, fmt.Errorf("unknown SCOS version %q", os.version) + } + + // We've been able to infer the necessary fields for FCOS and SCOS. + return os, nil } // Determines the OS version based upon the contents of the RHEL_VERSION, VERSION or VERSION_ID fields. diff --git a/pkg/daemon/osrelease/osrelease_test.go b/pkg/daemon/osrelease/osrelease_test.go index 637f166a66..62e8cedb1e 100644 --- a/pkg/daemon/osrelease/osrelease_test.go +++ b/pkg/daemon/osrelease/osrelease_test.go @@ -214,6 +214,283 @@ OSTREE_VERSION='37.20230126.20.0'` assert.Equal(t, testCase.IsSCOS, os.IsSCOS(), "expected IsSCOS() to be %v", testCase.IsSCOS) assert.Equal(t, testCase.IsLikeTraditionalRHEL7, os.IsLikeTraditionalRHEL7(), "expected IsLikeTraditionalRHEL7() to be %v", testCase.IsLikeTraditionalRHEL7) assert.Equal(t, testCase.ToPrometheusLabel, os.ToPrometheusLabel(), "expected ToPrometheusLabel() to be %s, got %s", testCase.ToPrometheusLabel, os.ToPrometheusLabel()) + assert.Equal(t, OSReleaseInfoSource, os.Source()) + for key, value := range os.Values() { + if value != "" { + assert.Contains(t, testCase.OSReleaseContents, key) + assert.Contains(t, testCase.OSReleaseContents, value) + } + } + }) + } +} + +func TestOSReleaseFromImageLabels(t *testing.T) { + t.Parallel() + + // $ skopeo inspect --no-tags "docker://$(oc adm release info --pullspecs "quay.io/openshift-release-dev/ocp-release:4.12.3-x86_64" -o json | jq -r '.references.spec.tags[] | select(.name == "rhel-coreos-8") | .from.name')" | jq '.Labels' + rhcos86ImageLabels := map[string]string{ + "coreos-assembler.image-config-checksum": "1b48fcfc2329e3b643345459fa579d094ab4cf32bbe02e6af7f75bed2d795664", + "coreos-assembler.image-input-checksum": "958ec627288ced8977d29fe56b753fddccb22b5574674dd466ca36ee0a65ef7b", + "org.opencontainers.image.revision": "87d66dba922c2b82490f582d3b40042892fd71d9", + "org.opencontainers.image.source": "https://github.com/openshift/os", + "ostree.bootable": "true", + "ostree.commit": "f6f57efc27907e9cb02b25abe8f662c83e5518573f8ac72db56d352f0e53132a", + "ostree.final-diffid": "sha256:dc44336569c692fdab109180b59f6cfd33d4276d7c463b1939a453c547407e91", + "ostree.linux": "4.18.0-372.43.1.el8_6.x86_64", + "rpmostree.inputhash": "d22d3ff9f440dd92013e08b273feaf2ca1c7add9d86989d1ac5cb39914810fbe", + "version": "412.86.202302091057-0", + } + + // $ skopeo inspect --no-tags 'docker://registry.ci.openshift.org/ocp/4.13:rhel-coreos-9' | jq '.Labels' + rhcos92ImageLabels := map[string]string{ + "coreos-assembler.image-config-checksum": "01bebb5709ab54cef0e7bac1f94e6ee333b25868ab022f8618cdb4443de99355", + "coreos-assembler.image-input-checksum": "cd8d504aea915c32fce40e0dbf3c2424cf9612df6e21a00874668abb8ab75c10", + "org.opencontainers.image.revision": "d717505b88821990fee2d96688c337995d849ecd", + "org.opencontainers.image.source": "https://github.com/openshift/os", + "ostree.bootable": "true", + "ostree.commit": "75a403bab1c8e65291f8b905d6080789a123961d341b56a3b8988cf87fa50ee4", + "ostree.final-diffid": "sha256:896666c665dc31ed85f0cad0a23b6d8fed5fdcb913c5d3f21d2157a3c3c72883", + "ostree.linux": "5.14.0-252.el9.x86_64", + "rpmostree.inputhash": "ff03479ce560a2e7d33518f56151c798de8af5cec9c80fc6849586f2a46df30d", + "version": "413.92.202302081904-0", + } + + // $ skopeo inspect --no-tags "docker://$(oc adm release info --pullspecs "registry.ci.openshift.org/origin/release-scos:4.13.0-0.okd-scos-2023-02-13-084859" -o json | jq -r '.references.spec.tags[] | select(.name == "centos-stream-coreos-9") | .from.name')" | jq '.Labels' + scosImageLabels := map[string]string{ + "coreos-assembler.image-config-checksum": "60de8e2e7e531654866c9b4cb39f25ab40f7c1c6a228c67094532582fff32517", + "coreos-assembler.image-input-checksum": "242aa41d018c8898be12f52920ced133d8c113663a73f73d3f9ff3f4cba616ec", + "io.openshift.build.version-display-names": "machine-os=CentOS Stream CoreOS", + "io.openshift.build.versions": "machine-os=413.9.202302130811-0", + "org.opencontainers.image.revision": "0d059dc913d80034c4947c83b45cab9134a4b76b", + "org.opencontainers.image.source": "https://github.com/openshift/os.git", + "ostree.bootable": "true", + "ostree.commit": "bf556ab10cfd284ce9af1760bb3c58e021eeb53e3dad0a7067fc27cc00005229", + "ostree.final-diffid": "sha256:6bab3208d1cc275dde2344b7b821b2c42c2bb9839316d9f8e7dec7856f3ce95b", + "ostree.linux": "5.14.0-252.el9.x86_64", + "rpmostree.inputhash": "b0a41cc20b028c5480fef2d4fc91c1b26d3b99cdef18e3812e6c290f7f4bb942", + "version": "413.9.202302130811-0", + } + + // $ skopeo inspect --no-tags "docker://$(oc adm release info --pullspecs "registry.ci.openshift.org/origin/release:4.13.0-0.okd-2023-02-13-013048" -o json | jq -r '.references.spec.tags[] | select(.name == "fedora-coreos") | .from.name')" | jq '.Labels' + fcosImageLabels := map[string]string{ + "coreos-assembler.image-config-checksum": "12fcea940941bb0fdfc7e693af5d5b80c20103cad5b2f7d1b325695c23e267bb", + "coreos-assembler.image-input-checksum": "545e10a3b7b9678dee912f65946bae68115239c77060427e56c9be36725822e4", + "fedora-coreos.stream": "testing-devel", + "io.buildah.version": "1.26.4", + "io.openshift.build.commit.author": "", + "io.openshift.build.commit.date": "", + "io.openshift.build.commit.id": "dca734c49f2d3b72877059d0efe4b0ebb46bf0cf", + "io.openshift.build.commit.message": "", + "io.openshift.build.commit.ref": "master", + "io.openshift.build.name": "", + "io.openshift.build.namespace": "", + "io.openshift.build.source-context-dir": "", + "io.openshift.build.source-location": "https://github.com/openshift/okd-machine-os", + "io.openshift.build.version-display-names": "machine-os=Fedora CoreOS", + "io.openshift.build.versions": "machine-os=37.20230211.20", + "io.openshift.release.operator": "true", + "org.opencontainers.image.revision": "93c169993547dacc9a3db20c5aa9b010edc9b7fe", + "org.opencontainers.image.source": "https://github.com/coreos/fedora-coreos-config", + "ostree.bootable": "true", + "ostree.commit": "a9cae909177102060c848b4088c046274fcfb8297ff1544bd80b0c6c8ce43f47", + "ostree.final-diffid": "sha256:e6fbb4f4b828c4cefb957fd625b1205c15615303a82f019db1f66c80901ca08b", + "ostree.linux": "6.1.10-200.fc37.x86_64", + "rpmostree.inputhash": "4042bcea14f8713abf293302c26b60bca32438e6b59c7ec96761f6b28a0762b2", + "vcs-ref": "dca734c49f2d3b72877059d0efe4b0ebb46bf0cf", + "vcs-type": "git", + "vcs-url": "https://github.com/openshift/okd-machine-os", + "version": "37.20230211.20.0", + } + + // $ skopeo inspect --no-tags 'docker://registry.access.redhat.com/ubi8/ubi:latest' | jq '.Labels' + ubi8ImageLabels := map[string]string{ + "architecture": "x86_64", + "build-date": "2023-02-07T17:57:16", + "com.redhat.component": "ubi8-container", + "com.redhat.license_terms": "https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI", + "description": "The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.", + "distribution-scope": "public", + "io.buildah.version": "1.27.3", + "io.k8s.description": "The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.", + "io.k8s.display-name": "Red Hat Universal Base Image 8", + "io.openshift.expose-services": "", + "io.openshift.tags": "base rhel8", + "maintainer": "Red Hat, Inc.", + "name": "ubi8", + "release": "1054.1675788412", + "summary": "Provides the latest release of Red Hat Universal Base Image 8.", + "url": "https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.7-1054.1675788412", + "vcs-ref": "a995512a05037e3b60bbb1bf9fa6e394063131c3", + "vcs-type": "git", + "vendor": "Red Hat, Inc.", + "version": "8.7", + } + + // $ skopeo inspect --no-tags 'docker://registry.access.redhat.com/ubi9/ubi:latest' | jq '.Labels' + ubi9ImageLabels := map[string]string{ + "architecture": "x86_64", + "build-date": "2023-02-07T16:24:49", + "com.redhat.component": "ubi9-container", + "com.redhat.license_terms": "https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI", + "description": "The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.", + "distribution-scope": "public", + "io.buildah.version": "1.27.3", + "io.k8s.description": "The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.", + "io.k8s.display-name": "Red Hat Universal Base Image 9", + "io.openshift.expose-services": "", + "io.openshift.tags": "base rhel9", + "maintainer": "Red Hat, Inc.", + "name": "ubi9", + "release": "1750.1675784955", + "summary": "Provides the latest release of Red Hat Universal Base Image 9.", + "url": "https://access.redhat.com/containers/#/registry.access.redhat.com/ubi9/images/9.1.0-1750.1675784955", + "vcs-ref": "cf87ad00feaef3d9d7a442dad55ab6a14f6a3f81", + "vcs-type": "git", + "vendor": "Red Hat, Inc.", + "version": "9.1.0", + } + + testCases := []struct { + Name string + OSImageLabels map[string]string + IsEL bool + IsEL9 bool + IsFCOS bool + IsSCOS bool + IsCoreOSVariant bool + IsLikeTraditionalRHEL7 bool + ToPrometheusLabel string + ErrorExpected bool + }{ + { + Name: "RHCOS 8.6", + OSImageLabels: rhcos86ImageLabels, + IsEL: true, + IsEL9: false, + IsFCOS: false, + IsSCOS: false, + IsCoreOSVariant: true, + IsLikeTraditionalRHEL7: false, + ToPrometheusLabel: "RHCOS", + }, + { + Name: "RHCOS 9.2", + OSImageLabels: rhcos92ImageLabels, + IsEL: true, + IsEL9: true, + IsFCOS: false, + IsSCOS: false, + IsCoreOSVariant: true, + IsLikeTraditionalRHEL7: false, + ToPrometheusLabel: "RHCOS", + }, + { + Name: "SCOS", + OSImageLabels: scosImageLabels, + IsEL: true, + IsEL9: true, + IsFCOS: false, + IsSCOS: true, + IsCoreOSVariant: true, + IsLikeTraditionalRHEL7: false, + ToPrometheusLabel: "SCOS", + }, + { + Name: "FCOS", + OSImageLabels: fcosImageLabels, + IsEL: false, + IsEL9: false, + IsFCOS: true, + IsSCOS: false, + IsCoreOSVariant: true, + IsLikeTraditionalRHEL7: false, + ToPrometheusLabel: "FEDORA", + }, + { + Name: "Unidentifiable OS - Unknown Name", + OSImageLabels: map[string]string{ + "coreos-assembler.image-input-checksum": "", + "coreos-assembler.image-config-checksum": "", + "org.opencontainers.image.revision": "", + "org.opencontainers.image.source": "", + "version": "", + "io.openshift.build.version-display-names": "machine-os=Unknown Operating System", + }, + ErrorExpected: true, + }, + { + Name: "Unidentifiable OS - Empty labels", + OSImageLabels: map[string]string{}, + ErrorExpected: true, + }, + { + Name: "Unidentifiable OS - Invalid RHCOS Version ID", + OSImageLabels: map[string]string{ + "coreos-assembler.image-input-checksum": "", + "coreos-assembler.image-config-checksum": "", + "org.opencontainers.image.revision": "", + "org.opencontainers.image.source": "", + "version": "37.20230211.20.0", + }, + ErrorExpected: true, + }, + { + Name: "Unidentifiable OS - Invalid SCOS Version ID", + OSImageLabels: map[string]string{ + "coreos-assembler.image-input-checksum": "", + "coreos-assembler.image-config-checksum": "", + "org.opencontainers.image.revision": "", + "org.opencontainers.image.source": "", + "version": "37.20230211.20.0", + "io.openshift.build.version-display-names": "machine-os=CentOS Stream CoreOS", + }, + ErrorExpected: true, + }, + { + Name: "Unidentifiable OS - Fedora 37 Container", + OSImageLabels: map[string]string{ + // $ skopeo inspect --no-tags 'docker://registry.fedoraproject.org/fedora:latest' | jq '.Labels' + "license": "MIT", + "name": "fedora", + "vendor": "Fedora Project", + "version": "37", + }, + ErrorExpected: true, + }, + { + Name: "Unidentifiable OS - Red Hat UBI 8", + OSImageLabels: ubi8ImageLabels, + ErrorExpected: true, + }, + { + Name: "Unidentifiable OS - Red Hat UBI 9", + OSImageLabels: ubi9ImageLabels, + ErrorExpected: true, + }, + } + + for _, testCase := range testCases { + testCase := testCase + t.Run(testCase.Name, func(t *testing.T) { + t.Parallel() + os, err := InferFromOSImageLabels(testCase.OSImageLabels) + if testCase.ErrorExpected { + assert.Error(t, err) + return + } + + assert.NoError(t, err) + + assert.Equal(t, testCase.IsEL, os.IsEL(), "expected IsEL() to be %v", testCase.IsEL) + assert.Equal(t, testCase.IsEL9, os.IsEL9(), "expected IsEL9() to be %v", testCase.IsEL9) + assert.Equal(t, testCase.IsCoreOSVariant, os.IsCoreOSVariant(), "expected IsCoreOSVariant() to be %v", testCase.IsCoreOSVariant) + assert.Equal(t, testCase.IsFCOS, os.IsFCOS(), "expected IsFCOS() to be %v", testCase.IsFCOS) + assert.Equal(t, testCase.IsSCOS, os.IsSCOS(), "expected IsSCOS() to be %v", testCase.IsSCOS) + assert.Equal(t, testCase.IsLikeTraditionalRHEL7, os.IsLikeTraditionalRHEL7(), "expected IsLikeTraditionalRHEL7() to be %v", testCase.IsLikeTraditionalRHEL7) + assert.Equal(t, testCase.ToPrometheusLabel, os.ToPrometheusLabel(), "expected ToPrometheusLabel() to be %s, got %s", testCase.ToPrometheusLabel, os.ToPrometheusLabel()) + assert.Equal(t, ImageLabelInfoSource, os.Source()) + assert.Equal(t, testCase.OSImageLabels, os.Values()) }) } } diff --git a/pkg/daemon/update.go b/pkg/daemon/update.go index 68dc36febb..e5e7b198d2 100644 --- a/pkg/daemon/update.go +++ b/pkg/daemon/update.go @@ -16,6 +16,7 @@ import ( "time" "github.com/clarketm/json" + "github.com/containers/image/v5/types" ign3types "github.com/coreos/ignition/v2/config/v3_2/types" "github.com/golang/glog" corev1 "k8s.io/api/core/v1" @@ -27,6 +28,7 @@ import ( mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common" "github.com/openshift/machine-config-operator/pkg/daemon/constants" + "github.com/openshift/machine-config-operator/pkg/daemon/osrelease" pivottypes "github.com/openshift/machine-config-operator/pkg/daemon/pivot/types" pivotutils "github.com/openshift/machine-config-operator/pkg/daemon/pivot/utils" ) @@ -1644,9 +1646,86 @@ func (dn *Daemon) updateSSHKeys(newUsers []ign3types.PasswdUser) error { return nil } +// Gets the labels attached to the OS image and uses that to infer the OS version. +func (dn *Daemon) getNewOSVersion(imgURL string) (osr osrelease.OperatingSystem, err error) { + glog.Infof("Looking up image labels for new OS %q to infer new OS version", imgURL) + + // The image inspection code is copy / pasted from the Rebase() method in rpm-ostree.go. + var imageData *types.ImageInspectInfo + if imageData, _, err = imageInspect(imgURL); err != nil { + if err != nil { + var podmanImgData *imageInspection + glog.Infof("Falling back to using podman inspect") + if podmanImgData, err = podmanInspect(imgURL); err != nil { + return + } + defer exec.Command("podman", "rmi", imgURL).Run() + + osr, err = osrelease.InferFromOSImageLabels(podmanImgData.Labels) + return + } + } + + osr, err = osrelease.InferFromOSImageLabels(imageData.Labels) + return +} + +// Returns true if we're updating from RHCOS8 -> RHCOS9. Returns false for all +// other cases. +func (dn *Daemon) isRHCOS9Upgrade(newImageOS osRelease) bool { + return dn.os.IsEL() && !dn.os.IsEL9() && newImageOS.IsEL() && newImageOS.IsEL9() +} + +// Prepares for an upgrade from RHCOS 8 to RHCOS 9 by doing the following: +// - Installing a newer version of rpm-ostree from the extensions container. +// - Restarting rpm-ostree.service. +// +// Assumes: +// - We've called ExtractExtensionsImage +// - We've called addExtensionsRepo or addLayeredExtensionsRepo. +func (dn *Daemon) prepForRHCOS9(newImageURL string) error { + // Gets the image labels for the new OS image and infers what the OS version + // is based upon their contents. + newOSInfo, err := dn.getNewOSVersion(newImageURL) + if err != nil { + return fmt.Errorf("could not get new OS version info: %w", err) + } + + // If we don't have an RHCOS 8 -> RHCOS 9 upgrade, return early. + if !dn.isRHCOS9Upgrade(newOSInfo) { + return nil + } + + glog.Infof("Detected RHCOS 8 -> RHCOS 9 upgrade, upgrading rpm-ostree first") + + // Ensures that the extensions repo is available first. + if _, err := os.Stat(extensionsRepo); err != nil { + return fmt.Errorf("extensions repo not found: %w", err) + } + + // Upgrades rpm-ostree. + // TODO(zzlotnik): Is this the right command to upgrade rpm-ostree? + if _, err := runGetOut("rpm", "-Uvh", "rpm-ostree"); err != nil { + return fmt.Errorf("could not install updated rpm-ostree: %w", err) + } + + // Restarts the rpm-ostreed service. + glog.Info("Restarting rpm-ostree service") + if _, err := runGetOut("systemd", "restart", "rpm-ostreed.service"); err != nil { + return fmt.Errorf("could not restart rpm-ostreed: %w", err) + } + + return nil +} + // updateOS updates the system OS to the one specified in newConfig func (dn *Daemon) updateOS(config *mcfgv1.MachineConfig, osImageContentDir string) error { newURL := config.Spec.OSImageURL + + if err := dn.prepForRHCOS9(newURL); err != nil { + return fmt.Errorf("could not prepare for RHCOS 9 upgrade: %w", err) + } + glog.Infof("Updating OS to %s", newURL) if _, err := dn.NodeUpdaterClient.Rebase(newURL, osImageContentDir); err != nil { return fmt.Errorf("failed to update OS to %s : %w", newURL, err) @@ -1719,6 +1798,7 @@ func (dn *Daemon) updateLayeredOS(config *mcfgv1.MachineConfig) error { if err != nil { return err } + // If the host isn't new enough to understand the new container model natively, run as a privileged container. // See https://github.com/coreos/rpm-ostree/pull/3961 and https://issues.redhat.com/browse/MCO-356 // This currently will incur a double reboot; see https://github.com/coreos/rpm-ostree/issues/4018 @@ -1727,7 +1807,13 @@ func (dn *Daemon) updateLayeredOS(config *mcfgv1.MachineConfig) error { if err := dn.InplaceUpdateViaNewContainer(newURL); err != nil { return err } - } else if err := dn.NodeUpdaterClient.RebaseLayered(newURL); err != nil { + } + + if err := dn.prepForRHCOS9(newURL); err != nil { + return fmt.Errorf("could not prepare for RHCOS 9 upgrade: %w", err) + } + + if err := dn.NodeUpdaterClient.RebaseLayered(newURL); err != nil { return fmt.Errorf("failed to update OS to %s : %w", newURL, err) } diff --git a/pkg/daemon/update_test.go b/pkg/daemon/update_test.go index cb724168b2..abc7b051ef 100644 --- a/pkg/daemon/update_test.go +++ b/pkg/daemon/update_test.go @@ -16,7 +16,6 @@ import ( ign3types "github.com/coreos/ignition/v2/config/v3_2/types" mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common" - "github.com/openshift/machine-config-operator/pkg/daemon/osrelease" "github.com/openshift/machine-config-operator/test/helpers" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -25,12 +24,79 @@ import ( k8sfake "k8s.io/client-go/kubernetes/fake" ) +type fakeOS struct { + isEL bool + isEL9 bool + isFCOS bool + isSCOS bool + isCoreOSVariant bool + isLikeTraditionalRHEL7 bool +} + +func (f fakeOS) IsEL() bool { + return f.isEL +} + +func (f fakeOS) IsEL9() bool { + return f.isEL9 +} + +func (f fakeOS) IsFCOS() bool { + return f.isFCOS +} + +func (f fakeOS) IsSCOS() bool { + return f.isSCOS +} + +func (f fakeOS) IsCoreOSVariant() bool { + return f.isCoreOSVariant +} + +func (f fakeOS) IsLikeTraditionalRHEL7() bool { + return f.isLikeTraditionalRHEL7 +} + +func rhcos8() fakeOS { + return fakeOS{ + isEL: true, + isEL9: false, + isCoreOSVariant: true, + } +} + +func rhcos9() fakeOS { + return fakeOS{ + isEL: true, + isEL9: true, + isCoreOSVariant: true, + } +} + +func fcos() fakeOS { + return fakeOS{ + isFCOS: true, + isCoreOSVariant: true, + } +} + +func scos() fakeOS { + return fakeOS{ + isEL: true, + isEL9: true, + isSCOS: true, + isCoreOSVariant: true, + } +} + +var _ osRelease = fakeOS{} + func newMockDaemon() Daemon { // Create a Daemon instance with mocked clients return Daemon{ mock: true, name: "nodeName", - os: osrelease.OperatingSystem{}, + os: fakeOS{}, kubeClient: k8sfake.NewSimpleClientset(), bootedOSImageURL: "test", } @@ -752,3 +818,55 @@ func TestOriginalFileBackupRestore(t *testing.T) { assert.Nil(t, err) } + +func TestRHCOS8to9Update(t *testing.T) { + testCases := []struct { + name string + nodeOS osRelease + imageOS osRelease + isRHCOS9Upgrade bool + }{ + { + name: "With FCOS base", + nodeOS: fcos(), + imageOS: fcos(), + isRHCOS9Upgrade: false, + }, + { + name: "With SCOS base", + nodeOS: scos(), + imageOS: scos(), + isRHCOS9Upgrade: false, + }, + { + name: "With RHCOS8 base", + nodeOS: rhcos8(), + imageOS: rhcos9(), + isRHCOS9Upgrade: true, + }, + { + name: "With RHCOS8 base to RHCOS8 new OS", + nodeOS: rhcos8(), + imageOS: rhcos8(), + isRHCOS9Upgrade: false, + }, + { + name: "With RHCOS9 base", + nodeOS: rhcos9(), + imageOS: rhcos9(), + isRHCOS9Upgrade: false, + }, + } + + for _, testCase := range testCases { + testCase := testCase + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + d := newMockDaemon() + d.os = testCase.nodeOS + + assert.Equal(t, d.isRHCOS9Upgrade(testCase.imageOS), testCase.isRHCOS9Upgrade) + }) + } +} diff --git a/test/e2e/node_os_test.go b/test/e2e/node_os_test.go index 5d5b90e0bf..051b147118 100644 --- a/test/e2e/node_os_test.go +++ b/test/e2e/node_os_test.go @@ -2,18 +2,27 @@ package e2e import ( "context" + "encoding/json" + "fmt" "strings" "testing" + imagestreamv1 "github.com/openshift/api/image/v1" + "github.com/openshift/machine-config-operator/pkg/daemon/osrelease" "github.com/openshift/machine-config-operator/test/framework" "github.com/openshift/machine-config-operator/test/helpers" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" ) const ( + // These patterns are intentionally left unterminated so they match all RHCOS 8 / 9 versions. + rhelVersion8 string = "RHEL_VERSION=\"8" + rhelVersion9 string = "RHEL_VERSION=\"9" + rhcos string = "Red Hat Enterprise Linux CoreOS" scos string = "CentOS Stream CoreOS" fcos string = "Fedora Linux" @@ -22,11 +31,16 @@ const ( // Verifies that the OS on each node is identifiable using the mechanism the // MCD does. This is done to ensure that we get an early warning if the // contents of /etc/os-release or /usr/lib/os-release unexpectedly change. +// +// We do the same for the labels attached to the OS image associated with the +// release. This is done for the same early warning. func TestOSDetection(t *testing.T) { cs := framework.NewClientSet("") nodes, err := cs.CoreV1Interface.Nodes().List(context.TODO(), metav1.ListOptions{}) require.NoError(t, err) + osImageRelease, osImageReleaseSrc := getOSReleaseInfoForOSImage(t, cs, nodes.Items[0]) + for _, node := range nodes.Items { node := node t.Run("", func(t *testing.T) { @@ -39,12 +53,24 @@ func TestOSDetection(t *testing.T) { assert.False(t, nodeOSRelease.OS.IsLikeTraditionalRHEL7(), "expected IsLikeTraditionalRHEL7() to be false: %s", nodeOSRelease.EtcContent) switch { - case strings.Contains(nodeOSRelease.EtcContent, rhcos): - assertRHCOS(t, nodeOSRelease, node) + case strings.Contains(nodeOSRelease.EtcContent, rhcos) && strings.Contains(nodeOSRelease.EtcContent, rhelVersion8): + rhelVersion := nodeOSRelease.OS.Values()["RHEL_VERSION"] + assertIsRHCOS8(t, nodeOSRelease.OS, nodeOSRelease.EtcContent) + assertIsRHCOS8(t, osImageRelease, osImageReleaseSrc) + t.Logf("Identified %s %s on node %s", rhcos, rhelVersion, node.Name) + case strings.Contains(nodeOSRelease.EtcContent, rhcos) && strings.Contains(nodeOSRelease.EtcContent, rhelVersion9): + rhelVersion := nodeOSRelease.OS.Values()["RHEL_VERSION"] + assertIsRHCOS9(t, nodeOSRelease.OS, nodeOSRelease.EtcContent) + assertIsRHCOS9(t, osImageRelease, osImageReleaseSrc) + t.Logf("Identified %s %s on node %s", rhcos, rhelVersion, node.Name) case strings.Contains(nodeOSRelease.EtcContent, scos): - assertSCOS(t, nodeOSRelease, node) + assertIsSCOS(t, nodeOSRelease.OS, nodeOSRelease.EtcContent) + assertIsSCOS(t, osImageRelease, osImageReleaseSrc) + t.Logf("Identified %s on node %s", scos, node.Name) case strings.Contains(nodeOSRelease.EtcContent, fcos): - assertFCOS(t, nodeOSRelease, node) + assertIsFCOS(t, nodeOSRelease.OS, nodeOSRelease.EtcContent) + assertIsFCOS(t, osImageRelease, osImageReleaseSrc) + t.Logf("Identified %s on node %s", fcos, node.Name) default: t.Fatalf("unknown OS on node %s detected: %s", node.Name, nodeOSRelease.EtcContent) } @@ -52,35 +78,119 @@ func TestOSDetection(t *testing.T) { } } -func assertRHCOS(t *testing.T, nodeOSRelease helpers.NodeOSRelease, node corev1.Node) { - assert.True(t, nodeOSRelease.OS.IsEL(), "expected IsEL() to be true: %s", nodeOSRelease.EtcContent) - assert.False(t, nodeOSRelease.OS.IsSCOS(), "expected IsSCOS() to be false: %s", nodeOSRelease.EtcContent) - assert.False(t, nodeOSRelease.OS.IsFCOS(), "expected IsFCOS() to be false: %s", nodeOSRelease.EtcContent) +func assertIsRHCOS(t *testing.T, osr osrelease.OperatingSystem, source string) { + t.Helper() - rhelVersion := nodeOSRelease.OS.OSRelease().ADDITIONAL_FIELDS["RHEL_VERSION"] + assert.True(t, osr.IsEL(), "expected IsEL() to be true: %s", source) + assert.False(t, osr.IsSCOS(), "expected IsSCOS() to be false: %s", source) + assert.False(t, osr.IsFCOS(), "expected IsFCOS() to be false: %s", source) +} - if strings.Contains(nodeOSRelease.EtcContent, "RHEL_VERSION=\"8.") { // This pattern intentionally unterminated so it matches all RHCOS 8 versions - t.Logf("Identified %s %s on node %s", rhcos, rhelVersion, node.Name) - assert.False(t, nodeOSRelease.OS.IsEL9(), "expected < RHCOS 9.0: %s", nodeOSRelease.EtcContent) - } +func assertIsRHCOS8(t *testing.T, osr osrelease.OperatingSystem, source string) { + t.Helper() - if strings.Contains(nodeOSRelease.EtcContent, "RHEL_VERSION=\"9.") { // This pattern intentionally unterminated so it matches all RHCOS 9 versions - t.Logf("Identified %s %s on node %s", rhcos, rhelVersion, node.Name) - assert.True(t, nodeOSRelease.OS.IsEL9(), "expected >= RHCOS 9.0+: %s", nodeOSRelease.EtcContent) - } + assertIsRHCOS(t, osr, source) + assert.False(t, osr.IsEL9(), "expected < RHCOS 9.0: %s", source) } -func assertSCOS(t *testing.T, nodeOSRelease helpers.NodeOSRelease, node corev1.Node) { - t.Logf("Identified %s on node %s", scos, node.Name) - assert.True(t, nodeOSRelease.OS.IsEL(), "expected IsEL() to be true: %s", nodeOSRelease.EtcContent) - assert.True(t, nodeOSRelease.OS.IsEL9(), "expected IsEL9() to be true: %s", nodeOSRelease.EtcContent) - assert.False(t, nodeOSRelease.OS.IsFCOS(), "expected IsFCOS() to be false: %s", nodeOSRelease.EtcContent) +func assertIsRHCOS9(t *testing.T, osr osrelease.OperatingSystem, source string) { + t.Helper() + + assertIsRHCOS(t, osr, source) + assert.True(t, osr.IsEL9(), "expected >= RHCOS 9.0+: %s", source) } -func assertFCOS(t *testing.T, nodeOSRelease helpers.NodeOSRelease, node corev1.Node) { - t.Logf("Identified OS %s on node %s", fcos, node.Name) - assert.False(t, nodeOSRelease.OS.IsEL(), "expected IsEL() to be false: %s", nodeOSRelease.EtcContent) - assert.False(t, nodeOSRelease.OS.IsEL9(), "expected IsEL9() to be false: %s", nodeOSRelease.EtcContent) - assert.False(t, nodeOSRelease.OS.IsSCOS(), "expected IsSCOS() to be false: %s", nodeOSRelease.EtcContent) - assert.True(t, nodeOSRelease.OS.IsFCOS(), "expected IsFCOS() to be true: %s", nodeOSRelease.EtcContent) +func assertIsFCOS(t *testing.T, osr osrelease.OperatingSystem, source string) { + t.Helper() + + assert.False(t, osr.IsEL(), "expected IsEL() to be false: %s", source) + assert.False(t, osr.IsEL9(), "expected IsEL9() to be false: %s", source) + assert.False(t, osr.IsSCOS(), "expected IsSCOS() to be false: %s", source) + assert.True(t, osr.IsFCOS(), "expected IsFCOS() to be true: %s", source) +} + +func assertIsSCOS(t *testing.T, osr osrelease.OperatingSystem, source string) { + assert.True(t, osr.IsEL(), "expected IsEL() to be true: %s", source) + assert.True(t, osr.IsEL9(), "expected IsEL9() to be true: %s", source) + assert.False(t, osr.IsFCOS(), "expected IsFCOS() to be false: %s", source) +} + +// Gets the image labels from the relevant OS image from the cluster release. +// There is probably a better way to do this, but this seemed to be the easiest +// that would work across multiple testing contexts (e.g., local developer / CI / etc.) +func getOSReleaseInfoForOSImage(t *testing.T, cs *framework.ClientSet, node corev1.Node) (osrelease.OperatingSystem, string) { + t.Helper() + + authFile := "/var/lib/kubelet/config.json" + + version, err := cs.ClusterVersions().Get(context.TODO(), "version", metav1.GetOptions{}) + require.NoError(t, err) + + // Runs oc adm release info on a given node to get the release ImageStream. + // We're only interested in the references object, so we use jsonpath to + // limit output to just that. This is equivalent to: + // $ oc adm release info "release-image" -o json | jq '.references' + rawImagestream := helpers.ExecCmdOnNode(t, cs, node, []string{ + "chroot", + "/rootfs", + "oc", + "adm", + "release", + "info", + "--registry-config", + authFile, + "-o=jsonpath={.references}", + version.Status.Desired.Image, + }...) + + releaseImagestream := imagestreamv1.ImageStream{} + + require.NoError(t, json.Unmarshal([]byte(rawImagestream), &releaseImagestream)) + + // These are the known OS image tag names. + knownTags := sets.NewString("rhel-coreos-8", "rhel-coreos-9", "fedora-coreos", "centos-stream-coreos-9") + knownTagMatch := "" + osImagePullspec := "" + + // Locate the image pullspec based upon the known tag name. + for _, tag := range releaseImagestream.Spec.Tags { + if knownTags.Has(tag.Name) { + knownTagMatch = tag.Name + osImagePullspec = tag.From.Name + break + } + } + + if osImagePullspec == "" { + t.Fatalf("couldn't find a known OS image pullspec in release %s; known OS image pullspecs: %v", version.Status.Desired.Image, knownTags.List()) + } else { + t.Logf("Found OS image %q matching tag %q in OCP / OKD release image %q. Will compare to what is on cluster nodes", osImagePullspec, knownTagMatch, version.Status.Desired.Image) + } + + // Use skopeo on the node to get the labels for the OS image pullspec. + // This is equivalent to: + // $ skopeo inspect docker:// | jq '.Labels' + rawImageLabels := helpers.ExecCmdOnNode(t, cs, node, []string{ + "chroot", + "/rootfs", + "skopeo", + "inspect", + "--no-tags", + "--authfile", + authFile, + fmt.Sprintf("docker://%s", osImagePullspec), + }...) + + type imageLabels struct { + Labels map[string]string + } + + labels := imageLabels{} + + require.NoError(t, json.Unmarshal([]byte(rawImageLabels), &labels)) + + osr, err := osrelease.InferFromOSImageLabels(labels.Labels) + require.NoError(t, err) + + return osr, fmt.Sprintf("Labels for OS image %q: %v", osImagePullspec, labels.Labels) }