diff --git a/doozerlib/assembly.py b/doozerlib/assembly.py index fe20b192f..672b35fd0 100644 --- a/doozerlib/assembly.py +++ b/doozerlib/assembly.py @@ -21,6 +21,7 @@ class AssemblyIssueCode(Enum): OUTDATED_RPMS_IN_STREAM_BUILD = 4 INCONSISTENT_RHCOS_RPMS = 5 MISSING_INHERITED_DEPENDENCY = 6 + MISSING_RHCOS_CONTAINER = 7 class AssemblyIssue: diff --git a/doozerlib/assembly_inspector.py b/doozerlib/assembly_inspector.py index e532e5413..3c01d84c4 100644 --- a/doozerlib/assembly_inspector.py +++ b/doozerlib/assembly_inspector.py @@ -8,7 +8,7 @@ from doozerlib.rpmcfg import RPMMetadata from doozerlib.assembly import assembly_rhcos_config, AssemblyTypes, assembly_permits, AssemblyIssue, \ AssemblyIssueCode, assembly_type -from doozerlib.rhcos import RHCOSBuildInspector, RHCOSBuildFinder +from doozerlib.rhcos import RHCOSBuildInspector, RHCOSBuildFinder, get_container_configs, RhcosMissingContainerException class AssemblyInspector: @@ -309,17 +309,34 @@ def get_rhcos_build(self, arch: str, private: bool = False, custom: bool = False brew_arch = util.brew_arch_for_go_arch(arch) runtime.logger.info(f"Getting latest RHCOS source for {brew_arch}...") - # See if this assembly has assembly.rhcos.machine-os-content.images populated for this architecture. - assembly_rhcos_arch_pullspec = self.assembly_rhcos_config['machine-os-content'].images[brew_arch] + # See if this assembly has assembly.rhcos.*.images populated for this architecture. + pullspec_for_tag = dict() + for container_conf in get_container_configs(runtime): + # first we look at the assembly definition as the source of truth for RHCOS containers + assembly_rhcos_arch_pullspec = self.assembly_rhcos_config[container_conf.name].images[brew_arch] + if assembly_rhcos_arch_pullspec: + pullspec_for_tag[container_conf.name] = assembly_rhcos_arch_pullspec + continue - if self.runtime.assembly_type != AssemblyTypes.STREAM and not assembly_rhcos_arch_pullspec: - raise Exception(f'Assembly {runtime.assembly} has is not a STREAM but no assembly.rhcos MOSC image data for {brew_arch}; all MOSC image data must be populated for this assembly to be valid') + # for non-stream assemblies we expect explicit config for RHCOS + if self.runtime.assembly_type != AssemblyTypes.STREAM: + if container_conf.primary: + raise Exception(f'Assembly {runtime.assembly} is not type STREAM but no assembly.rhcos.{container_conf.name} image data for {brew_arch}; all RHCOS image data must be populated for this assembly to be valid') + # require the primary container at least to be specified, but + # allow the edge case where we add an RHCOS container type and + # previous assemblies don't specify it + continue - version = self.runtime.get_minor_version() - if assembly_rhcos_arch_pullspec: - return RHCOSBuildInspector(runtime, assembly_rhcos_arch_pullspec, brew_arch) - else: - _, pullspec = RHCOSBuildFinder(runtime, version, brew_arch, private, custom=custom).latest_machine_os_content() - if not pullspec: - raise IOError(f"No RHCOS latest found for {version} / {brew_arch}") - return RHCOSBuildInspector(runtime, pullspec, brew_arch) + try: + version = self.runtime.get_minor_version() + _, pullspec = RHCOSBuildFinder(runtime, version, brew_arch, private, custom=custom).latest_container(container_conf) + if not pullspec: + raise IOError(f"No RHCOS latest found for {version} / {brew_arch}") + pullspec_for_tag[container_conf.name] = pullspec + except RhcosMissingContainerException: + if container_conf.primary: + # accommodate RHCOS build metadata not specifying all expected containers, but require primary. + # their absence will be noted when generating payloads anyway. + raise + + return RHCOSBuildInspector(runtime, pullspec_for_tag, brew_arch) diff --git a/doozerlib/cli/detect_embargo.py b/doozerlib/cli/detect_embargo.py index e51d909dc..b73b7efff 100644 --- a/doozerlib/cli/detect_embargo.py +++ b/doozerlib/cli/detect_embargo.py @@ -7,7 +7,7 @@ import click import yaml -from doozerlib import Runtime, brew, exectools +from doozerlib import Runtime, brew, exectools, rhcos from doozerlib import build_status_detector as bs_detector from doozerlib.cli import cli, pass_runtime from doozerlib.exceptions import DoozerFatalError @@ -204,8 +204,12 @@ def detect_embargoes_in_releases(runtime: Runtime, pullspecs: List[str]): :return: list of Brew build dicts that have embargoed fixes """ runtime.logger.info(f"Fetching component pullspecs from {len(pullspecs)} release payloads...") - jobs = runtime.parallel_exec(lambda pullspec, _: get_image_pullspecs_from_release_payload(pullspec), pullspecs, - min(len(pullspecs), multiprocessing.cpu_count() * 4, 32)) + ignore_rhcos_tags = rhcos.get_container_names(runtime) + jobs = runtime.parallel_exec( + lambda pullspec, _: get_image_pullspecs_from_release_payload(pullspec, ignore_rhcos_tags), + pullspecs, + min(len(pullspecs), multiprocessing.cpu_count() * 4, 32) + ) pullspec_lists = jobs.get() embargoed_releases = [] embargoed_pullspecs = [] @@ -272,7 +276,7 @@ def get_nvr_by_pullspec(pullspec: str) -> Tuple[str, str, str]: return (labels.get("com.redhat.component"), labels.get("version"), labels.get("release")) -def get_image_pullspecs_from_release_payload(payload_pullspec: str, ignore={"machine-os-content"}) -> Iterable[str]: +def get_image_pullspecs_from_release_payload(payload_pullspec: str, ignore=set()) -> Iterable[str]: """ Retrieves pullspecs of images in a release payload. :param payload_pullspec: release payload pullspec :param ignore: a set of image names that we want to exclude from the return value (e.g. machine-os-content) diff --git a/doozerlib/cli/get_nightlies.py b/doozerlib/cli/get_nightlies.py index e1f6f4e94..394545545 100644 --- a/doozerlib/cli/get_nightlies.py +++ b/doozerlib/cli/get_nightlies.py @@ -4,10 +4,7 @@ from urllib import request from doozerlib.cli import cli from doozerlib import constants, util, exectools - -# See https://github.com/openshift/machine-config-operator/blob/master/docs/OSUpgrades.md -# But in the future this will be replaced, see https://github.com/coreos/enhancements/blob/main/os/coreos-layering.md -OLD_FORMAT_COREOS_TAG = 'machine-os-content' +from doozerlib.rhcos import get_primary_container_name @cli.command("get-nightlies", short_help="Get sets of Accepted nightlies. A set contains nightly for each arch, " @@ -40,6 +37,8 @@ def ignore_arch(arch): phase = '' nightlies_with_phase[arch] = all_nightlies_in_phase(major, minor, arch, phase) + primary_coreos_tag = get_primary_container_name(runtime) + i = 0 for x64_nightly, x64_phase in nightlies_with_phase["amd64"]: if i >= limit: @@ -56,7 +55,7 @@ def ignore_arch(arch): nightly_str = f'{nightly} {phase}' if rhcos: if phase != 'Pending': - rhcos = get_coreos_build_from_payload(get_nightly_pullspec(nightly, arch)) + rhcos = get_coreos_build_from_payload(get_nightly_pullspec(nightly, arch), primary_coreos_tag) nightly_str += f' {rhcos}' print(nightly_str) print(",".join(nightly_set)) @@ -68,9 +67,9 @@ def get_nightly_pullspec(release, arch): return f'registry.ci.openshift.org/ocp{suffix}/release{suffix}:{release}' -def get_coreos_build_from_payload(payload_pullspec): - """Retrive the build version of machine-os-content (e.g. 411.86.202206131434-0)""" - out, err = exectools.cmd_assert(["oc", "adm", "release", "info", "--image-for", OLD_FORMAT_COREOS_TAG, "--", payload_pullspec]) +def get_coreos_build_from_payload(payload_pullspec, primary_coreos_tag): + """Retrieve the build version of RHCOS (e.g. 411.86.202206131434-0)""" + out, err = exectools.cmd_assert(["oc", "adm", "release", "info", "--image-for", primary_coreos_tag, "--", payload_pullspec]) if err: raise Exception(f"Error running oc adm: {err}") rhcos_pullspec = out.split('\n')[0] diff --git a/doozerlib/cli/release_gen_assembly.py b/doozerlib/cli/release_gen_assembly.py index e2a2b75cc..d2ca794bc 100644 --- a/doozerlib/cli/release_gen_assembly.py +++ b/doozerlib/cli/release_gen_assembly.py @@ -13,6 +13,7 @@ from doozerlib import exectools from doozerlib.model import Model from doozerlib import brew +from doozerlib import rhcos from doozerlib.rpmcfg import RPMMetadata from doozerlib.image import BrewBuildImageInspector from doozerlib.runtime import Runtime @@ -103,7 +104,7 @@ def exit_with_error(msg): final_previous_list: List[VersionInfo] = sorted(final_previous_list) reference_releases_by_arch: Dict[str, str] = dict() # Maps brew arch name to nightly name - mosc_by_arch: Dict[str, str] = dict() # Maps brew arch name to machine-os-content pullspec from nightly + rhcos_by_tag: Dict[str, Dict[str, str]] = dict() # Maps RHCOS container name(s) to brew arch name to pullspec(s) from nightly component_image_builds: Dict[str, BrewBuildImageInspector] = dict() # Maps component package_name to brew build dict found for nightly component_rpm_builds: Dict[str, Dict[int, Dict]] = dict() # Dict[ package_name ] -> Dict[ el? ] -> brew build dict basis_event_ts: float = 0.0 @@ -130,6 +131,7 @@ def exit_with_error(msg): raise ValueError(f'Cannot process {standard_release_name} since {release_pullspecs[brew_cpu_arch]} is already included') release_pullspecs[brew_cpu_arch] = standard_pullspec + rhcos_tag_names = rhcos.get_container_names(runtime) for brew_cpu_arch, pullspec in release_pullspecs.items(): runtime.logger.info(f'Processing release: {pullspec}') @@ -143,8 +145,8 @@ def exit_with_error(msg): payload_tag_name = component_tag.name # e.g. "aws-ebs-csi-driver" payload_tag_pullspec = component_tag['from'].name # quay pullspec - if payload_tag_name == 'machine-os-content': - mosc_by_arch[brew_cpu_arch] = payload_tag_pullspec + if payload_tag_name in rhcos_tag_names: + rhcos_by_tag.setdefault(payload_tag_name, {})[brew_cpu_arch] = payload_tag_pullspec continue # The brew_build_inspector will take this archive image and find the actual @@ -184,7 +186,7 @@ def exit_with_error(msg): basis_event_ts = max(basis_event_ts, completion_ts + (60.0 * 5)) # basis_event_ts should now be greater than the build completion / target tagging operation - # for any (non machine-os-content) image in the nightlies. Because images are built after RPMs, + # for any (non RHCOS) image in the nightlies. Because images are built after RPMs, # it must also hold that the basis_event_ts is also greater than build completion & tagging # of any member RPM. @@ -253,15 +255,16 @@ def exit_with_error(msg): # image NVR does not need to be pinned. Yeah! pass - # We should have found a machine-os-content for each architecture in the group for a standard assembly + # We should have found an RHCOS container for each architecture in the group for a standard assembly + primary_rhcos_tag = rhcos.get_primary_container_name(runtime) for arch in runtime.arches: - if arch not in mosc_by_arch: + if arch not in rhcos_by_tag[primary_rhcos_tag]: if custom: # This is permitted for custom assemblies which do not need to be assembled for every # architecture. The customer may just need x86_64. - logger.info(f'Did not find machine-os-content image for active group architecture: {arch}; ignoring since this is custom.') + logger.info(f'Did not find RHCOS "{primary_rhcos_tag}" image for active group architecture: {arch}; ignoring for custom assembly type.') else: - exit_with_error(f'Did not find machine-os-content image for active group architecture: {arch}') + exit_with_error(f'Did not find RHCOS "{primary_rhcos_tag}" image for active group architecture: {arch}') # We now have a list of image builds that should be selected by the assembly basis event # and those that will need to be forced with 'is'. We now need to perform a similar step @@ -355,7 +358,7 @@ def exit_with_error(msg): # If the user has specified fewer nightlies than is required by this # group, then we need to override the group arches. group_info = { - 'arches!': list(mosc_by_arch.keys()) + 'arches!': list(rhcos_by_tag[primary_rhcos_tag].keys()) } if assembly_type not in [AssemblyTypes.CUSTOM, AssemblyTypes.PREVIEW]: # Add placeholder advisory numbers and JIRA key. @@ -382,9 +385,8 @@ def exit_with_error(msg): }, 'group': group_info, 'rhcos': { - 'machine-os-content': { - "images": mosc_by_arch, - } + tag: dict(images={arch: pullspec for arch, pullspec in specs_by_arch.items()}) + for tag, specs_by_arch in rhcos_by_tag.items() }, 'members': { 'rpms': rpm_member_overrides, diff --git a/doozerlib/cli/release_gen_payload.py b/doozerlib/cli/release_gen_payload.py index 576eac0e1..f07d1aa61 100644 --- a/doozerlib/cli/release_gen_payload.py +++ b/doozerlib/cli/release_gen_payload.py @@ -12,7 +12,7 @@ from doozerlib.rpm_utils import parse_nvr from doozerlib.brew import KojiWrapper -from doozerlib.rhcos import RHCOSBuildInspector +from doozerlib.rhcos import RHCOSBuildInspector, RhcosMissingContainerException from doozerlib.cli import cli, pass_runtime from doozerlib.image import ImageMetadata, BrewBuildImageInspector, ArchiveImageInspector from doozerlib.assembly_inspector import AssemblyInspector @@ -89,7 +89,7 @@ def modify_and_replace_api_object(api_obj: oc.APIObject, modifier_func: Callable api_obj.replace() -@cli.command("release:gen-payload", short_help="Generate input files for release mirroring") +@cli.command("release:gen-payload", short_help="Mirror release images to quay and release-controller") @click.option("--is-name", metavar='NAME', required=False, help="ImageStream .metadata.name value. For example '4.2-art-latest'") @click.option("--is-namespace", metavar='NAMESPACE', required=False, @@ -109,19 +109,19 @@ def modify_and_replace_api_object(api_obj: oc.APIObject, modifier_func: Callable @click.option("--emergency-ignore-issues", default=False, is_flag=True, help="If you must get this command to permit an assembly despite issues. Do not use without approval.") @click.option("--apply", default=False, is_flag=True, - help="If doozer should perform the mirroring and imagestream updates.") + help="Perform mirroring and imagestream updates.") @click.option("--apply-multi-arch", default=False, is_flag=True, - help="If doozer should also create a release payload for multi-arch/heterogeneous clusters.") + help="Also create a release payload for multi-arch/heterogeneous clusters.") @click.option("--moist-run", default=False, is_flag=True, - help="Performing mirroring/etc but to not actually update imagestreams.") + help="Mirror and determine tags but do not actually update imagestreams.") @pass_runtime def release_gen_payload(runtime: Runtime, is_name: str, is_namespace: str, organization: str, repository: str, release_repository: str, output_dir: str, exclude_arch: Tuple[str, ...], skip_gc_tagging: bool, emergency_ignore_issues: bool, apply: bool, apply_multi_arch: bool, moist_run: bool): """Computes a set of imagestream tags which can be assembled -into an OpenShift release for this assembly. The tags will not be -valid unless --apply and is supplied. +into an OpenShift release for this assembly. The tags may not be +valid unless --apply or --moist-run triggers mirroring. Applying the change will cause the OSBS images to be mirrored into the OpenShift release repositories on quay. @@ -143,9 +143,8 @@ def release_gen_payload(runtime: Runtime, is_name: str, is_namespace: str, organ --is-name=4.2-art-latest Note that if you use -i to include specific images, you should also include -openshift-enterprise-cli to satisfy any need for the 'cli' tag. The cli image -is used automatically as a stand-in for images when an arch does not build -that particular tag. +openshift-enterprise-pod to supply the 'pod' tag. The 'pod' image is used +automatically as a payload stand-in for images that do not build on all arches. ## Validation ## @@ -155,7 +154,7 @@ def release_gen_payload(runtime: Runtime, is_name: str, is_namespace: str, organ * For all architectures built, RHCOS builds must have matching versions of any unshipped RPM they include (per-entry os metadata - the set of RPMs may differ between arches, but versions should not). -* Any RPMs present in images (including machine-os-content) from unshipped RPM +* Any RPMs present in images (including RHCOS) from unshipped RPM builds included in one of our candidate tags must exactly version-match the latest RPM builds in those candidate tags (ONLY; we never flag what we don't directly ship.) @@ -320,8 +319,11 @@ def release_gen_payload(runtime: Runtime, is_name: str, is_namespace: str, organ continue # Whether private or public, the assembly's canonical payload content is the same. - entries: Dict[str, PayloadGenerator.PayloadEntry] = PayloadGenerator.find_payload_entries(assembly_inspector, arch, f'quay.io/{organization}/{repository}') # Key of this dict is release payload tag name + entries: Dict[str, PayloadGenerator.PayloadEntry] + issues: List[AssemblyIssue] + entries, issues = PayloadGenerator.find_payload_entries(assembly_inspector, arch, f'quay.io/{organization}/{repository}') # Key of this dict is release payload tag name entries_by_arch[arch] = entries + assembly_issues.extend(issues) for tag, payload_entry in entries.items(): if payload_entry.image_meta: @@ -401,7 +403,7 @@ def release_gen_payload(runtime: Runtime, is_name: str, is_namespace: str, organ with src_dest_path.open("w+", encoding="utf-8") as out_file: for payload_entry in entries.values(): if not payload_entry.archive_inspector: - # Nothing to mirror (e.g. machine-os-content) + # Nothing to mirror (e.g. RHCOS) continue def add_image_to_mirror(src_pullspec, dest_pullspec): @@ -587,7 +589,7 @@ def update_single_arch_istags(apiobj: oc.APIObject): # 1. The images for ALL arches were part of the same brew built manifest list. In this case, we # want to reuse the manifest list (it was already mirrored during the mirroring step). # 2. At least one arch for this component does not have the same manifest list as the - # other images. This will always be true for machine-os-content, but also applies + # other images. This will always be true for RHCOS, but also applies # to -alt images. In this case, we must stitch a manifest list together ourselves. aggregate_issues: List[AssemblyIssue] = list() @@ -792,7 +794,7 @@ class PayloadEntry(NamedTuple): # The final quay.io destination for the manifest list the single arch image # might belong to. Most images built in brew will have been part of a - # manifest list, but not all release components (e.g. machine-os-content) + # manifest list, but not all release components (e.g. RHCOS) # will be. We reuse manifest lists where possible for heterogeneous # release payloads to save time vs building them ourselves. dest_manifest_list_pullspec: str = None @@ -808,7 +810,7 @@ class PayloadEntry(NamedTuple): archive_inspector: Optional[ArchiveImageInspector] = None """ - If the entry is for machine-os-content, this value will be set + If the entry is for RHCOS, this value will be set """ rhcos_build: Optional[RHCOSBuildInspector] = None @@ -895,16 +897,24 @@ def get_mirroring_destination(sha256: str, dest_repo: str) -> str: tag = sha256.replace(":", "-") # sha256:abcdef -> sha256-abcdef return f"{dest_repo}:{tag}" - @staticmethod - def find_payload_entries(assembly_inspector: AssemblyInspector, arch: str, dest_repo: str) -> Dict[str, PayloadEntry]: + @classmethod + def find_payload_entries(clazz, assembly_inspector: AssemblyInspector, arch: str, dest_repo: str) -> (Dict[str, PayloadEntry], List[AssemblyIssue]): """ Returns a list of images which should be included in the architecture specific release payload. - This includes images for our group's image metadata as well as machine-os-content. + This includes images for our group's image metadata as well as RHCOS. :param assembly_inspector: An analyzer for the assembly to generate entries for. :param arch: The brew architecture name to create the list for. :param dest_repo: The registry/org/repo into which the image should be mirrored. :return: Map[payload_tag_name] -> PayloadEntry. """ + members: Dict[str, PayloadGenerator.PayloadEntry] = clazz._find_initial_payload_entries(assembly_inspector, arch, dest_repo) + members = clazz._replace_missing_payload_entries(members, arch) + rhcos_members, issues = clazz._find_rhcos_payload_entries(assembly_inspector, arch) + members.update(rhcos_members) + return members, issues + + @staticmethod + def _find_initial_payload_entries(assembly_inspector: AssemblyInspector, arch: str, dest_repo: str) -> Dict[str, PayloadEntry]: members: Dict[str, Optional[PayloadGenerator.PayloadEntry]] = dict() # Maps release payload tag name to the PayloadEntry for the image. for payload_tag, archive_inspector in PayloadGenerator.get_group_payload_tag_mapping(assembly_inspector, arch).items(): if not archive_inspector: @@ -921,36 +931,56 @@ def find_payload_entries(assembly_inspector: AssemblyInspector, arch: str, dest_ dest_manifest_list_pullspec=PayloadGenerator.get_mirroring_destination(archive_inspector.get_brew_build_inspector().get_manifest_list_digest(), dest_repo), issues=list(), ) + return members - # members now contains a complete map of payload tag keys, but some values may be None. This is an - # indication that the architecture did not have a build of one of our group images. - # The tricky bit is that all architecture specific release payloads contain the same set of tags - # or 'oc adm release new' will have trouble assembling it. i.e. an imagestream tag 'X' may not be - # necessary on s390x, bit we need to populate that tag with something. + @staticmethod + def _replace_missing_payload_entries(members: Dict[str, PayloadEntry], arch: str) -> Dict[str, PayloadEntry]: + # members contains a complete map of payload tag keys, but some values may be None, + # indicating that the image does not build for this architecture. + # However, all architecture-specific release payloads must contain the + # full set of tags or 'oc adm release new' will fail; while a tag may + # not be logically necessary on e.g. s390x, we still need to populate + # that tag with something for metadata references to resolve. # To do this, we replace missing images with the 'pod' image for the architecture. This should - # be available for every CPU architecture. As such, we must find pod to proceed. + # be available for every CPU architecture. As such, we must find 'pod' to proceed. pod_entry = members.get('pod', None) if not pod_entry: - raise IOError(f'Unable to find pod image archive for architecture: {arch}; unable to construct payload') + raise IOError(f"Unable to find 'pod' image archive for architecture: {arch}; unable to construct payload") - final_members: Dict[str, PayloadGenerator.PayloadEntry] = dict() - for tag_name, entry in members.items(): - if entry: - final_members[tag_name] = entry - else: - final_members[tag_name] = pod_entry + return { + tag_name: entry or pod_entry + for tag_name, entry in members.items() + } + @staticmethod + def _find_rhcos_payload_entries(assembly_inspector: AssemblyInspector, arch: str) -> (Dict[str, PayloadEntry], List[AssemblyIssue]): + members: Dict[str, PayloadGenerator.PayloadEntry] = dict() + issues: List[AssemblyIssue] = list() rhcos_build: RHCOSBuildInspector = assembly_inspector.get_rhcos_build(arch) - final_members['machine-os-content'] = PayloadGenerator.PayloadEntry( - dest_pullspec=rhcos_build.get_image_pullspec(), - rhcos_build=rhcos_build, - issues=list(), - ) + for container_config in rhcos_build.get_container_configs(): + try: + members[container_config.name] = PayloadGenerator.PayloadEntry( + dest_pullspec=rhcos_build.get_container_pullspec(container_config), + rhcos_build=rhcos_build, + issues=list(), + ) + except RhcosMissingContainerException as ex: + if container_config.primary: + # Impermissible, need to be sure of having the primary container in the payload + issues.append(AssemblyIssue( + f'RHCOS build {rhcos_build} metadata lacks entry for primary container {container_config.name}: {ex}', + component=container_config.name + )) + else: + issues.append(AssemblyIssue( + f'RHCOS build {rhcos_build} metadata lacks entry for non-primary container {container_config.name}: {ex}', + component=container_config.name, + code=AssemblyIssueCode.MISSING_RHCOS_CONTAINER + )) - # Final members should have all tags populated. - return final_members + return members, issues @staticmethod def build_payload_istag(payload_tag_name: str, payload_entry: PayloadEntry) -> Dict: @@ -1017,7 +1047,7 @@ def get_group_payload_tag_mapping(assembly_inspector: AssemblyInspector, arch: s Each payload tag name used to map exactly to one release imagemeta. With the advent of '-alt' images, we need some logic to determine which images map to which payload tags for a given architecture. :return: Returns a map[payload_tag_name] -> ArchiveImageInspector containing an image for the payload. The value may be - None if there is no arch specific build for the tag. This does not include machine-os-content since that + None if there is no arch specific build for the tag. This does not include RHCOS since that is not a member of the group. """ brew_arch = brew_arch_for_go_arch(arch) # Make certain this is brew arch nomenclature @@ -1099,8 +1129,9 @@ def terminal_issue(msg: str) -> List[AssemblyIssue]: if not release_info.references.spec.tags: return terminal_issue(f'Could not find tags in nightly {nightly}') - issues: List[AssemblyIssue] = list() - payload_entries: Dict[str, PayloadGenerator.PayloadEntry] = PayloadGenerator.find_payload_entries(assembly_inspector, arch, '') + payload_entries: Dict[str, PayloadGenerator.PayloadEntry] + issues: List[AssemblyIssue] + payload_entries, issues = PayloadGenerator.find_payload_entries(assembly_inspector, arch, '') for component_tag in release_info.references.spec.tags: # For each tag in the imagestream payload_tag_name: str = component_tag.name # e.g. "aws-ebs-csi-driver" payload_tag_pullspec: str = component_tag['from'].name # quay pullspec @@ -1120,9 +1151,9 @@ def terminal_issue(msg: str) -> List[AssemblyIssue]: issues.append(AssemblyIssue(f'{nightly} contains {payload_tag_name} sha {pullspec_sha} but assembly computed archive: {entry.archive_inspector.get_archive_id()} and {entry.archive_inspector.get_archive_pullspec()}', component='reference-releases')) elif entry.rhcos_build: - if entry.rhcos_build.get_machine_os_content_digest() != pullspec_sha: + if entry.rhcos_build.get_container_digest() != pullspec_sha: # Impermissible because the artist should remove the reference nightlies from the assembly definition - issues.append(AssemblyIssue(f'{nightly} contains {payload_tag_name} sha {pullspec_sha} but assembly computed rhcos: {entry.rhcos_build} and {entry.rhcos_build.get_machine_os_content_digest()}', + issues.append(AssemblyIssue(f'{nightly} contains {payload_tag_name} sha {pullspec_sha} but assembly computed rhcos: {entry.rhcos_build} and {entry.rhcos_build.get_container_digest()}', component='reference-releases')) else: raise IOError(f'Unsupported payload entry {entry}') diff --git a/doozerlib/cli/scan_sources.py b/doozerlib/cli/scan_sources.py index 42f62aa0c..2574db081 100644 --- a/doozerlib/cli/scan_sources.py +++ b/doozerlib/cli/scan_sources.py @@ -36,7 +36,7 @@ def config_scan_source_changes(runtime: Runtime, ci_kubeconfig, as_yaml): - If the associated member is a descendant of any image that needs change. \b - It will report machine-os-content updates available per imagestream. + It will report RHCOS updates available per imagestream. """ runtime.initialize(mode='both', clone_distgits=False, clone_source=False, prevent_cloning=True) @@ -251,7 +251,7 @@ def add_image_meta_change(meta, rebuild_hint: RebuildHint): def _detect_rhcos_status(runtime, kubeconfig) -> list: """ - gather the existing machine-os-content tags and compare them to latest rhcos builds + gather the existing RHCOS tags and compare them to latest rhcos builds @return a list of status entries like: { 'name': "4.2-x86_64-priv", @@ -265,30 +265,30 @@ def _detect_rhcos_status(runtime, kubeconfig) -> list: for arch in runtime.arches: for private in (False, True): name = f"{version}-{arch}{'-priv' if private else ''}" - tagged_mosc_id = _tagged_mosc_id(kubeconfig, version, arch, private) + tagged_rhcos_id = _tagged_rhcos_id(kubeconfig, rhcos.get_primary_container_name(runtime), version, arch, private) latest_rhcos_id = _latest_rhcos_build_id(runtime, version, arch, private) status = dict(name=name) if not latest_rhcos_id: status['changed'] = False status['reason'] = "could not find an RHCOS build to sync" - elif tagged_mosc_id == latest_rhcos_id: + elif tagged_rhcos_id == latest_rhcos_id: status['changed'] = False status['reason'] = f"latest RHCOS build is still {latest_rhcos_id} -- no change from istag" else: status['changed'] = True - status['reason'] = f"latest RHCOS build is {latest_rhcos_id} which differs from istag {tagged_mosc_id}" + status['reason'] = f"latest RHCOS build is {latest_rhcos_id} which differs from istag {tagged_rhcos_id}" statuses.append(status) return statuses -def _tagged_mosc_id(kubeconfig, version, arch, private) -> str: - """determine what the most recently tagged machine-os-content is in given imagestream""" +def _tagged_rhcos_id(kubeconfig, container_name, version, arch, private) -> str: + """determine the most recently tagged RHCOS in given imagestream""" base_name = rgp.default_imagestream_base_name(version) base_namespace = rgp.default_imagestream_namespace_base_name() name, namespace = rgp.payload_imagestream_name_and_namespace(base_name, base_namespace, arch, private) stdout, _ = exectools.cmd_assert( - f"oc --kubeconfig '{kubeconfig}' --namespace '{namespace}' get istag '{name}:machine-os-content'" + f"oc --kubeconfig '{kubeconfig}' --namespace '{namespace}' get istag '{name}:{container_name}'" " --template '{{.image.dockerImageMetadata.Config.Labels.version}}'", retries=3, pollrate=5, diff --git a/doozerlib/rhcos.py b/doozerlib/rhcos.py index a306ca79c..36b4cd488 100644 --- a/doozerlib/rhcos.py +++ b/doozerlib/rhcos.py @@ -9,18 +9,85 @@ from urllib.error import URLError from doozerlib.util import brew_suffix_for_arch, isolate_el_version_in_release from doozerlib import exectools -from doozerlib.model import Model, Missing +from doozerlib.model import ListModel, Model from doozerlib import brew RHCOS_BASE_URL = "https://rhcos-redirector.apps.art.xq1c.p1.openshiftapps.com/art/storage/releases" +# Historically the only RHCOS container was 'machine-os-content'; see +# https://github.com/openshift/machine-config-operator/blob/master/docs/OSUpgrades.md +# But in the future this will change, see +# https://github.com/coreos/enhancements/blob/main/os/coreos-layering.md +default_primary_container = dict( + name="machine-os-content", + build_metadata_key="oscontainer", + primary=True) -def rhcos_content_tag(runtime) -> str: +class RhcosMissingContainerException(Exception): """ - :return: Return the tag for packages we expect RHCOS to be built from. + Thrown when group.yml configuration expects an RHCOS container but it is + not available as specified in the RHCOS metadata. """ - base = runtime.group_config.branch.replace("-rhel-7", "-rhel-8") - return f"{base}-candidate" + pass + + +def get_container_configs(runtime): + """ + look up the group.yml configuration for RHCOS container(s) for this group, or create if missing. + @return ListModel with Model entries like ^^ default_primary_container + """ + return runtime.group_config.rhcos.payload_tags or ListModel([default_primary_container]) + + +def get_container_names(runtime): + """ + look up the payload tags of the group.yml-configured RHCOS container(s) for this group + @return list of container names + """ + return {tag.name for tag in get_container_configs(runtime)} + + +def get_primary_container_conf(runtime): + """ + look up the group.yml-configured primary RHCOS container for this group. + @return Model with entries for name and build_metadata_key + """ + for tag in get_container_configs(runtime): + if tag.primary: + return tag + raise Exception("Need to provide a group.yml rhcos.payload_tags entry with primary=true") + + +def get_primary_container_name(runtime): + """ + convenience method to retrieve configured primary RHCOS container name + @return primary container name (used in payload tag) + """ + return get_primary_container_conf(runtime).name + + +def get_container_pullspec(build_meta: dict, container_conf: Model) -> str: + """ + determine the container pullspec from the RHCOS build meta and config + @return full container pullspec string (registry/repo@sha256:...) + """ + key = container_conf.build_metadata_key + if key not in build_meta: + raise RhcosMissingContainerException(f"RHCOS build {build_meta['buildid']} has no '{key}' attribute in its metadata") + + container = build_meta[key] + + if 'digest' in container: + # "oscontainer": { + # "digest": "sha256:04b54950ce2...", + # "image": "quay.io/openshift-release-dev/ocp-v4.0-art-dev" + # }, + return container['image'] + "@" + container['digest'] + + # "base-oscontainer": { + # "image": "registry.ci.openshift.org/rhcos/rhel-coreos@sha256:b8e1064cae637f..." + # }, + return container['image'] class RHCOSNotFound(Exception): @@ -44,6 +111,16 @@ def __init__(self, runtime, version: str, brew_arch: str = "x86_64", private: bo self.brew_arch = brew_arch self.private = private self.custom = custom + self._primary_container = None + + def get_primary_container_conf(self): + """ + look up the group.yml-configured primary RHCOS container on demand and retain it. + @return Model with entries for name and build_metadata_key + """ + if not self._primary_container: + self._primary_container = get_primary_container_conf(self.runtime) + return self._primary_container def rhcos_release_url(self) -> str: """ @@ -122,40 +199,41 @@ def _rhcos_build_meta(self, build_id: str, meta_type: str = "meta") -> Dict: with request.urlopen(url) as req: return json.loads(req.read().decode()) - def latest_machine_os_content(self) -> Tuple[Optional[str], Optional[str]]: + def latest_container(self, container_conf: dict = None) -> Tuple[Optional[str], Optional[str]]: """ - :param version: The major.minor of the RHCOS stream the build is associated with (e.g. '4.6') - :param brew_arch: The CPU architecture for the build (uses brew naming convention) - :param private: Whether this is a private build (NOT CURRENTLY SUPPORTED) + :param container_conf: a payload tag conf Model from group.yml (with build_metadata_key) :return: Returns (rhcos build id, image pullspec) or (None, None) if not found. """ build_id = self.latest_rhcos_build_id() if build_id is None: return None, None - m_os_c = self.rhcos_build_meta(build_id)['oscontainer'] - return build_id, m_os_c['image'] + "@" + m_os_c['digest'] + return build_id, get_container_pullspec( + self.rhcos_build_meta(build_id), + container_conf or self.get_primary_container_conf() + ) class RHCOSBuildInspector: - def __init__(self, runtime, pullspec_or_build_id: str, brew_arch: str): + def __init__(self, runtime, pullspec_for_tag: Dict[str, str], brew_arch: str): self.runtime = runtime self.brew_arch = brew_arch - self.pullspec = None - - if pullspec_or_build_id[0].isdigit(): - self.build_id = pullspec_or_build_id - else: - # Remember the pullspec provided in case it does not match what is in the releases.yaml. - # Because of an incident where we needed to repush RHCOS and get a new SHA for 4.10 GA, - # trust the exact pullspec in releases.yml instead of what we find in the RHCOS release - # browser. - self.pullspec = pullspec_or_build_id - image_info_str, _ = exectools.cmd_assert(f'oc image info -o json {self.pullspec}', retries=3) - image_info = Model(dict_to_model=json.loads(image_info_str)) - self.build_id = image_info.config.config.Labels.version - if not self.build_id: - raise Exception(f'Unable to determine MOSC build_id from: {self.pullspec}. Retrieved image info: {image_info_str}') + self.pullspec_for_tag = pullspec_for_tag + self.build_id = None + + # Remember the pullspec(s) provided in case it does not match what is in the releases.yaml. + # Because of an incident where we needed to repush RHCOS and get a new SHA for 4.10 GA, + # trust the exact pullspec in releases.yml instead of what we find in the RHCOS release + # browser. + for tag, pullspec in pullspec_for_tag.items(): + image_info_str, _ = exectools.cmd_assert(f'oc image info -o json {pullspec}', retries=3) + image_info = Model(json.loads(image_info_str)) + build_id = image_info.config.config.Labels.version + if not build_id: + raise Exception(f'Unable to determine RHCOS build_id from tag {tag} pullspec {pullspec}. Retrieved image info: {image_info_str}') + if self.build_id and self.build_id != build_id: + raise Exception(f'Found divergent RHCOS build_id for {pullspec_for_tag}. {build_id} versus {self.build_id}') + self.build_id = build_id # The first digits of the RHCOS build are the major.minor of the rhcos stream name. # Which, near branch cut, might not match the actual release stream. @@ -167,7 +245,7 @@ def __init__(self, runtime, pullspec_or_build_id: str, brew_arch: str): finder = RHCOSBuildFinder(runtime, self.stream_version, self.brew_arch) self._build_meta = finder.rhcos_build_meta(self.build_id, meta_type='meta') self._os_commitmeta = finder.rhcos_build_meta(self.build_id, meta_type='commitmeta') - except: + except Exception: # Fall back to trying to find a custom build finder = RHCOSBuildFinder(runtime, self.stream_version, self.brew_arch, custom=True) self._build_meta = finder.rhcos_build_meta(self.build_id, meta_type='meta') @@ -242,12 +320,45 @@ def get_package_build_objects(self) -> Dict[str, Dict]: return aggregate - def get_image_pullspec(self) -> str: - if self.pullspec: - return self.pullspec - build_meta = self.get_build_metadata() - m_os_c = build_meta['oscontainer'] - return m_os_c['image'] + "@" + m_os_c['digest'] + def get_primary_container_conf(self): + """ + look up the group.yml-configured primary RHCOS container. + @return Model with entries for name and build_metadata_key + """ + return get_primary_container_conf(self.runtime) + + def get_container_configs(self): + """ + look up the group.yml-configured RHCOS containers and return their configs as a list + @return list(Model) with entries for name and build_metadata_key + """ + return get_container_configs(self.runtime) + + def get_container_pullspec(self, container_config: Model = None) -> str: + """ + Determine the pullspec corresponding to the container config given (the + primary by default), either as specified at instantiation or from the + build metadata. + + @param container_config: Model with fields "name" and "build_metadata_key" + :return: pullspec for the requested container image + """ + container_config = container_config or self.get_primary_container_conf() + if container_config.name in self.pullspec_for_tag: + # per note above... when given a pullspec, prefer that to the build record + return self.pullspec_for_tag[container_config.name] + return get_container_pullspec(self.get_build_metadata(), container_config) + + def get_container_digest(self, container_config: Model = None) -> str: + """ + Extract the image digest for (by default) the primary container image + associated with this build, historically the sha of the + machine-os-content image published out on quay. + + @param container_config: Model with fields "name" and "build_metadata_key" + :return: shasum from the pullspec for the requested container image + """ + return self.get_container_pullspec(container_config).split("@")[1] def get_rhel_base_version(self) -> int: """ @@ -263,13 +374,6 @@ def get_rhel_base_version(self) -> int: raise IOError(f'Unable to determine RHEL version base for rhcos {self.build_id}') - def get_machine_os_content_digest(self) -> str: - """ - Returns the image digest for the oscontainer image associated with this build. - This is the sha of the machine-os-content which should be published out on quay. - """ - return self._build_meta['oscontainer']['digest'] - def find_non_latest_rpms(self) -> List[Tuple[str, str]]: """ If the packages installed in this image overlap packages in the candidate tag, diff --git a/tests/cli/test_gen_payload.py b/tests/cli/test_gen_payload.py new file mode 100644 index 000000000..1b0ea2359 --- /dev/null +++ b/tests/cli/test_gen_payload.py @@ -0,0 +1,50 @@ +from unittest import TestCase +from unittest.mock import MagicMock, patch + +import yaml + +from doozerlib.assembly import AssemblyIssueCode +from doozerlib.cli import release_gen_payload as rgp_cli +from doozerlib.model import Model +from doozerlib import rhcos + + +class TestGenPayloadCli(TestCase): + + def test_find_rhcos_payload_entries(self): + rhcos_build = MagicMock() + assembly_inspector = MagicMock() + assembly_inspector.get_rhcos_build.return_value = rhcos_build + rhcos_build.get_container_configs.return_value = [ + Model(dict(name="spam", build_metadata_tag="eggs", primary=True)), + Model(dict(name="foo", build_metadata_tag="bar")), + ] + + # test when a primary container is missing from rhcos build + rhcos_build.get_container_pullspec.side_effect = [ + rhcos.RhcosMissingContainerException("primary missing"), + "somereg/somerepo@sha256:somesum", + ] + rhcos_entries, issues = rgp_cli.PayloadGenerator._find_rhcos_payload_entries(assembly_inspector, "arch") + self.assertNotIn("spam", rhcos_entries) + self.assertIn("foo", rhcos_entries) + self.assertEqual(issues[0].code, AssemblyIssueCode.IMPERMISSIBLE) + + # test when a non-primary container is missing from rhcos build + rhcos_build.get_container_pullspec.side_effect = [ + "somereg/somerepo@sha256:somesum", + rhcos.RhcosMissingContainerException("non-primary missing"), + ] + rhcos_entries, issues = rgp_cli.PayloadGenerator._find_rhcos_payload_entries(assembly_inspector, "arch") + self.assertIn("spam", rhcos_entries) + self.assertNotIn("foo", rhcos_entries) + self.assertEqual(issues[0].code, AssemblyIssueCode.MISSING_RHCOS_CONTAINER) + + # test when no container is missing from rhcos build + rhcos_build.get_container_pullspec.side_effect = [ + "somereg/somerepo@sha256:somesum", + "somereg/somerepo@sha256:someothersum", + ] + rhcos_entries, issues = rgp_cli.PayloadGenerator._find_rhcos_payload_entries(assembly_inspector, "arch") + self.assertEqual([], issues) + self.assertEqual(2, len(rhcos_entries)) diff --git a/tests/cli/test_scan_sources.py b/tests/cli/test_scan_sources.py index 0286ce53a..aa20da821 100644 --- a/tests/cli/test_scan_sources.py +++ b/tests/cli/test_scan_sources.py @@ -4,25 +4,26 @@ import yaml from doozerlib.cli import scan_sources +from doozerlib.model import Model from doozerlib import rhcos class TestScanSourcesCli(TestCase): @patch("doozerlib.exectools.cmd_assert") - def test_tagged_mosc_id(self, mock_cmd): + def test_tagged_rhcos_id(self, mock_cmd): mock_cmd.return_value = ("id-1", "stderr") - self.assertEqual("id-1", scan_sources._tagged_mosc_id("kc.conf", "4.2", "s390x", True)) + self.assertEqual("id-1", scan_sources._tagged_rhcos_id("kc.conf", "cname", "4.2", "s390x", True)) self.assertIn("--kubeconfig 'kc.conf'", mock_cmd.call_args_list[0][0][0]) self.assertIn("--namespace 'ocp-s390x-priv'", mock_cmd.call_args_list[0][0][0]) self.assertIn("istag '4.2-art-latest-s390x-priv", mock_cmd.call_args_list[0][0][0]) - @patch("doozerlib.cli.scan_sources._tagged_mosc_id", autospec=True) + @patch("doozerlib.cli.scan_sources._tagged_rhcos_id", autospec=True) @patch("doozerlib.cli.scan_sources._latest_rhcos_build_id", autospec=True) def test_detect_rhcos_status(self, mock_latest, mock_tagged): mock_tagged.return_value = "id-1" mock_latest.return_value = "id-2" - runtime = MagicMock() + runtime = MagicMock(group_config=Model()) runtime.get_minor_version.return_value = "4.2" runtime.arches = ['s390x'] diff --git a/tests/test_rhcos.py b/tests/test_rhcos.py index 4c2ece5d4..23f26bb8f 100755 --- a/tests/test_rhcos.py +++ b/tests/test_rhcos.py @@ -50,6 +50,14 @@ def setUp(self): def tearDown(self): pass + def test_get_primary_container_conf(self): + # default is same as it's always been + self.assertEqual("machine-os-content", rhcos.RHCOSBuildFinder(self.runtime, "4.6", "x86_64").get_primary_container_conf()["name"]) + + # but we can configure a different primary + self.runtime.group_config.rhcos = Model(dict(payload_tags=[dict(name="spam"), dict(name="eggs", primary=True)])) + self.assertEqual("eggs", rhcos.RHCOSBuildFinder(self.runtime, "4.6", "x86_64").get_primary_container_conf()["name"]) + def test_release_url(self): self.assertIn("4.6-s390x", rhcos.RHCOSBuildFinder(self.runtime, "4.6", "s390x").rhcos_release_url()) self.assertNotIn("x86_64", rhcos.RHCOSBuildFinder(self.runtime, "4.6", "x86_64").rhcos_release_url()) @@ -77,16 +85,30 @@ def test_build_find_failure(self, mock_urlopen): @patch('doozerlib.rhcos.RHCOSBuildFinder.latest_rhcos_build_id') @patch('doozerlib.rhcos.RHCOSBuildFinder.rhcos_build_meta') - def test_build_meta(self, meta_mock, id_mock): + def test_latest_container(self, meta_mock, id_mock): + # "normal" lookup id_mock.return_value = "dummy" meta_mock.return_value = dict(oscontainer=dict(image="test", digest="sha256:1234abcd")) - self.assertEqual(("dummy", "test@sha256:1234abcd"), rhcos.RHCOSBuildFinder(self.runtime, "4.4").latest_machine_os_content()) + self.assertEqual(("dummy", "test@sha256:1234abcd"), rhcos.RHCOSBuildFinder(self.runtime, "4.4").latest_container()) + # lookup when there is no build to look up id_mock.return_value = None - self.assertEqual((None, None), rhcos.RHCOSBuildFinder(self.runtime, "4.4").latest_machine_os_content()) + self.assertEqual((None, None), rhcos.RHCOSBuildFinder(self.runtime, "4.4").latest_container()) + # lookup when we have configured a different primary container + self.runtime.group_config.rhcos = Model(dict(payload_tags=[dict(name="spam"), dict(name="eggs", primary=True)])) + id_mock.return_value = "dummy" + meta_mock.return_value = dict( + oscontainer=dict(image="test", digest="sha256:1234abcdstandard"), + altcontainer=dict(image="test", digest="sha256:abcd1234alt"), + ) + alt_container = dict(name="rhel-coreos-8", build_metadata_key="altcontainer", primary=True) + self.runtime.group_config.rhcos = Model(dict(payload_tags=[alt_container])) + self.assertEqual(("dummy", "test@sha256:abcd1234alt"), rhcos.RHCOSBuildFinder(self.runtime, "4.4").latest_container()) + + @patch('doozerlib.exectools.cmd_assert') @patch('doozerlib.rhcos.RHCOSBuildFinder.rhcos_build_meta') - def test_rhcos_build_inspector(self, rhcos_build_meta_mock): + def test_rhcos_build_inspector(self, rhcos_build_meta_mock, cmd_assert_mock): """ Tests the RHCOS build inspector abstraction to ensure it correctly parses and utilizes pre-canned data. @@ -99,9 +121,14 @@ def test_rhcos_build_inspector(self, rhcos_build_meta_mock): pkg_build_dicts = yaml.safe_load(self.respath.joinpath('rhcos1', '47.83.202107261211-0.pkg_builds.yaml').read_text()) rhcos_build_meta_mock.side_effect = [rhcos_meta, rhcos_commitmeta] - rhcos_build = rhcos.RHCOSBuildInspector(self.runtime, '47.83.202107261211-0', 's390x') + cmd_assert_mock.return_value = ('{"config": {"config": {"Labels": {"version": "47.83.202107261211-0"}}}}', None) + test_digest = 'sha256:spamneggs' + test_pullspec = f'somereg/somerepo@{test_digest}' + pullspecs = {'machine-os-content': test_pullspec} + + rhcos_build = rhcos.RHCOSBuildInspector(self.runtime, pullspecs, 's390x') self.assertEqual(rhcos_build.brew_arch, 's390x') - self.assertEqual(rhcos_build.get_image_pullspec(), 'quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d51ca4e301cfdbc98e16ace0bcbee02b143a8be9e454ce5fb196467981141f59') + self.assertEqual(rhcos_build.get_container_pullspec(), test_pullspec) self.assertEqual(rhcos_build.stream_version, '4.7') self.assertEqual(rhcos_build.get_rhel_base_version(), 8) @@ -118,7 +145,23 @@ def canned_getBuild(build_id, *_, **__): self.assertIn("util-linux-2.32.1-24.el8.s390x", rhcos_build.get_rpm_nvras()) self.assertIn("util-linux-2.32.1-24.el8", rhcos_build.get_rpm_nvrs()) self.assertEqual(rhcos_build.get_package_build_objects()['dbus']['nvr'], 'dbus-1.12.8-12.el8_3') - self.assertEqual(rhcos_build.get_machine_os_content_digest(), 'sha256:d51ca4e301cfdbc98e16ace0bcbee02b143a8be9e454ce5fb196467981141f59') + self.assertEqual(rhcos_build.get_container_digest(), test_digest) + + @patch('doozerlib.exectools.cmd_assert') + @patch('doozerlib.rhcos.RHCOSBuildFinder.rhcos_build_meta') + def test_inspector_get_container_pullspec(self, rhcos_build_meta_mock, cmd_assert_mock): + # mock out the things RHCOSBuildInspector calls in __init__ + rhcos_meta = {"buildid": "412.86.bogus"} + rhcos_commitmeta = {} + rhcos_build_meta_mock.side_effect = [rhcos_meta, rhcos_commitmeta] + cmd_assert_mock.return_value = ('{"config": {"config": {"Labels": {"version": "412.86.bogus"}}}}', None) + pullspecs = {'machine-os-content': 'spam@eggs'} + rhcos_build = rhcos.RHCOSBuildInspector(self.runtime, pullspecs, 's390x') + + # test its behavior on misconfiguration / edge case + container_conf = dict(name='spam', build_metadata_key='eggs') + with self.assertRaises(rhcos.RhcosMissingContainerException): + rhcos_build.get_container_pullspec(Model(container_conf)) if __name__ == "__main__":