diff --git a/gangplank/cosa/cosa_v1.go b/gangplank/cosa/cosa_v1.go index f3086a4f44..fb1595171f 100644 --- a/gangplank/cosa/cosa_v1.go +++ b/gangplank/cosa/cosa_v1.go @@ -50,6 +50,7 @@ type Build struct { FedoraCoreOsParentVersion string `json:"fedora-coreos.parent-version,omitempty"` Gcp *Gcp `json:"gcp,omitempty"` GitDirty string `json:"coreos-assembler.config-dirty,omitempty"` + IbmCloud *Cloudartifact `json:"ibmcloud,omitempty"` ImageInputChecksum string `json:"coreos-assembler.image-input-checksum,omitempty"` InputHasOfTheRpmOstree string `json:"rpm-ostree-inputhash"` Koji *Koji `json:"koji,omitempty"` @@ -69,38 +70,42 @@ type Build struct { OverridesActive bool `json:"coreos-assembler.overrides-active,omitempty"` PkgdiffAgainstParent PackageSetDifferences `json:"parent-pkgdiff,omitempty"` PkgdiffBetweenBuilds PackageSetDifferences `json:"pkgdiff,omitempty"` + PowerVirtualServer *Cloudartifact `json:"powervs,omitempty"` ReleasePayload *Image `json:"release-payload,omitempty"` } type BuildArtifacts struct { - Aliyun *Artifact `json:"aliyun,omitempty"` - Aws *Artifact `json:"aws,omitempty"` - Azure *Artifact `json:"azure,omitempty"` - AzureStack *Artifact `json:"azurestack,omitempty"` - Dasd *Artifact `json:"dasd,omitempty"` - DigitalOcean *Artifact `json:"digitalocean,omitempty"` - Exoscale *Artifact `json:"exoscale,omitempty"` - Gcp *Artifact `json:"gcp,omitempty"` - IbmCloud *Artifact `json:"ibmcloud,omitempty"` - Initramfs *Artifact `json:"initramfs,omitempty"` - Iso *Artifact `json:"iso,omitempty"` - Kernel *Artifact `json:"kernel,omitempty"` - LiveInitramfs *Artifact `json:"live-initramfs,omitempty"` - LiveIso *Artifact `json:"live-iso,omitempty"` - LiveKernel *Artifact `json:"live-kernel,omitempty"` - LiveRootfs *Artifact `json:"live-rootfs,omitempty"` - Metal *Artifact `json:"metal,omitempty"` - Metal4KNative *Artifact `json:"metal4k,omitempty"` - OpenStack *Artifact `json:"openstack,omitempty"` - Ostree Artifact `json:"ostree"` - Qemu *Artifact `json:"qemu,omitempty"` - Vmware *Artifact `json:"vmware,omitempty"` - Vultr *Artifact `json:"vultr,omitempty"` + Aliyun *Artifact `json:"aliyun,omitempty"` + Aws *Artifact `json:"aws,omitempty"` + Azure *Artifact `json:"azure,omitempty"` + AzureStack *Artifact `json:"azurestack,omitempty"` + Dasd *Artifact `json:"dasd,omitempty"` + DigitalOcean *Artifact `json:"digitalocean,omitempty"` + Exoscale *Artifact `json:"exoscale,omitempty"` + Gcp *Artifact `json:"gcp,omitempty"` + IbmCloud *Artifact `json:"ibmcloud,omitempty"` + Initramfs *Artifact `json:"initramfs,omitempty"` + Iso *Artifact `json:"iso,omitempty"` + Kernel *Artifact `json:"kernel,omitempty"` + LiveInitramfs *Artifact `json:"live-initramfs,omitempty"` + LiveIso *Artifact `json:"live-iso,omitempty"` + LiveKernel *Artifact `json:"live-kernel,omitempty"` + LiveRootfs *Artifact `json:"live-rootfs,omitempty"` + Metal *Artifact `json:"metal,omitempty"` + Metal4KNative *Artifact `json:"metal4k,omitempty"` + OpenStack *Artifact `json:"openstack,omitempty"` + Ostree Artifact `json:"ostree"` + PowerVirtualServer *Artifact `json:"powervs,omitempty"` + Qemu *Artifact `json:"qemu,omitempty"` + Vmware *Artifact `json:"vmware,omitempty"` + Vultr *Artifact `json:"vultr,omitempty"` } type Cloudartifact struct { - Image string `json:"image"` - URL string `json:"url"` + Bucket string `json:"bucket,omitempty"` + Image string `json:"image"` + Region string `json:"region,omitempty"` + URL string `json:"url"` } type Extensions struct { diff --git a/mantle/cmd/ore/ibmcloud/upload.go b/mantle/cmd/ore/ibmcloud/upload.go index cb9a53054c..f5b6c49354 100644 --- a/mantle/cmd/ore/ibmcloud/upload.go +++ b/mantle/cmd/ore/ibmcloud/upload.go @@ -86,8 +86,7 @@ func runUpload(cmd *cobra.Command, args []string) error { // check if the s3 bucket exists // create s3 client - err = API.NewS3Client(cloudObjectStorage, region) - if err != nil { + if err := API.NewS3Client(uploadCloudObjectStorage, region); err != nil { return err } diff --git a/mantle/cosa/cosa_v1.go b/mantle/cosa/cosa_v1.go index f3086a4f44..fb1595171f 100644 --- a/mantle/cosa/cosa_v1.go +++ b/mantle/cosa/cosa_v1.go @@ -50,6 +50,7 @@ type Build struct { FedoraCoreOsParentVersion string `json:"fedora-coreos.parent-version,omitempty"` Gcp *Gcp `json:"gcp,omitempty"` GitDirty string `json:"coreos-assembler.config-dirty,omitempty"` + IbmCloud *Cloudartifact `json:"ibmcloud,omitempty"` ImageInputChecksum string `json:"coreos-assembler.image-input-checksum,omitempty"` InputHasOfTheRpmOstree string `json:"rpm-ostree-inputhash"` Koji *Koji `json:"koji,omitempty"` @@ -69,38 +70,42 @@ type Build struct { OverridesActive bool `json:"coreos-assembler.overrides-active,omitempty"` PkgdiffAgainstParent PackageSetDifferences `json:"parent-pkgdiff,omitempty"` PkgdiffBetweenBuilds PackageSetDifferences `json:"pkgdiff,omitempty"` + PowerVirtualServer *Cloudartifact `json:"powervs,omitempty"` ReleasePayload *Image `json:"release-payload,omitempty"` } type BuildArtifacts struct { - Aliyun *Artifact `json:"aliyun,omitempty"` - Aws *Artifact `json:"aws,omitempty"` - Azure *Artifact `json:"azure,omitempty"` - AzureStack *Artifact `json:"azurestack,omitempty"` - Dasd *Artifact `json:"dasd,omitempty"` - DigitalOcean *Artifact `json:"digitalocean,omitempty"` - Exoscale *Artifact `json:"exoscale,omitempty"` - Gcp *Artifact `json:"gcp,omitempty"` - IbmCloud *Artifact `json:"ibmcloud,omitempty"` - Initramfs *Artifact `json:"initramfs,omitempty"` - Iso *Artifact `json:"iso,omitempty"` - Kernel *Artifact `json:"kernel,omitempty"` - LiveInitramfs *Artifact `json:"live-initramfs,omitempty"` - LiveIso *Artifact `json:"live-iso,omitempty"` - LiveKernel *Artifact `json:"live-kernel,omitempty"` - LiveRootfs *Artifact `json:"live-rootfs,omitempty"` - Metal *Artifact `json:"metal,omitempty"` - Metal4KNative *Artifact `json:"metal4k,omitempty"` - OpenStack *Artifact `json:"openstack,omitempty"` - Ostree Artifact `json:"ostree"` - Qemu *Artifact `json:"qemu,omitempty"` - Vmware *Artifact `json:"vmware,omitempty"` - Vultr *Artifact `json:"vultr,omitempty"` + Aliyun *Artifact `json:"aliyun,omitempty"` + Aws *Artifact `json:"aws,omitempty"` + Azure *Artifact `json:"azure,omitempty"` + AzureStack *Artifact `json:"azurestack,omitempty"` + Dasd *Artifact `json:"dasd,omitempty"` + DigitalOcean *Artifact `json:"digitalocean,omitempty"` + Exoscale *Artifact `json:"exoscale,omitempty"` + Gcp *Artifact `json:"gcp,omitempty"` + IbmCloud *Artifact `json:"ibmcloud,omitempty"` + Initramfs *Artifact `json:"initramfs,omitempty"` + Iso *Artifact `json:"iso,omitempty"` + Kernel *Artifact `json:"kernel,omitempty"` + LiveInitramfs *Artifact `json:"live-initramfs,omitempty"` + LiveIso *Artifact `json:"live-iso,omitempty"` + LiveKernel *Artifact `json:"live-kernel,omitempty"` + LiveRootfs *Artifact `json:"live-rootfs,omitempty"` + Metal *Artifact `json:"metal,omitempty"` + Metal4KNative *Artifact `json:"metal4k,omitempty"` + OpenStack *Artifact `json:"openstack,omitempty"` + Ostree Artifact `json:"ostree"` + PowerVirtualServer *Artifact `json:"powervs,omitempty"` + Qemu *Artifact `json:"qemu,omitempty"` + Vmware *Artifact `json:"vmware,omitempty"` + Vultr *Artifact `json:"vultr,omitempty"` } type Cloudartifact struct { - Image string `json:"image"` - URL string `json:"url"` + Bucket string `json:"bucket,omitempty"` + Image string `json:"image"` + Region string `json:"region,omitempty"` + URL string `json:"url"` } type Extensions struct { diff --git a/mantle/cosa/schema_doc.go b/mantle/cosa/schema_doc.go index 406b1188af..2218e67c24 100644 --- a/mantle/cosa/schema_doc.go +++ b/mantle/cosa/schema_doc.go @@ -77,6 +77,10 @@ var generatedSchemaJSON = `{ "image", "url" ], + "optional": [ + "bucket", + "region" + ], "properties": { "image": { "$id":"#/cloudartifact/image", @@ -87,6 +91,16 @@ var generatedSchemaJSON = `{ "$id":"#/cloudartifact/url", "type":"string", "title":"URL" + }, + "bucket": { + "$id":"#/cloudartifact/bucket", + "type":"string", + "title":"Bucket" + }, + "region": { + "$id":"#/cloudartifact/region", + "type":"string", + "title":"Region" } } }, @@ -187,6 +201,7 @@ var generatedSchemaJSON = `{ "exoscale", "gcp", "ibmcloud", + "powervs", "images", "koji", "oscontainer", @@ -380,6 +395,7 @@ var generatedSchemaJSON = `{ "exoscale", "gcp", "ibmcloud", + "powervs", "initramfs", "iso", "kernel", @@ -527,6 +543,12 @@ var generatedSchemaJSON = `{ "title":"IBM Cloud", "$ref": "#/definitions/artifact" }, + "powervs": { + "$id":"#/properties/images/properties/powervs", + "type":"object", + "title":"Power Virtual Server", + "$ref": "#/definitions/artifact" + }, "gcp": { "$id":"#/properties/images/properties/gcp", "type":"object", @@ -797,6 +819,18 @@ var generatedSchemaJSON = `{ } } }, + "ibmcloud": { + "$id":"#/properties/ibmcloud", + "type":"object", + "title":"IBM Cloud", + "$ref": "#/definitions/cloudartifact" + }, + "powervs": { + "$id":"#/properties/powervs", + "type":"object", + "title":"Power Virtual Server", + "$ref": "#/definitions/cloudartifact" + }, "release-payload": { "$id":"#/properties/release-payload", "type":"object", diff --git a/src/cmd-buildextend-ibmcloud b/src/cmd-buildextend-ibmcloud index c0388001b2..4f0d225c78 120000 --- a/src/cmd-buildextend-ibmcloud +++ b/src/cmd-buildextend-ibmcloud @@ -1 +1 @@ -cmd-artifact-disk \ No newline at end of file +cmd-ore-wrapper \ No newline at end of file diff --git a/src/cmd-buildextend-powervs b/src/cmd-buildextend-powervs new file mode 120000 index 0000000000..4f0d225c78 --- /dev/null +++ b/src/cmd-buildextend-powervs @@ -0,0 +1 @@ +cmd-ore-wrapper \ No newline at end of file diff --git a/src/cmd-ore-wrapper b/src/cmd-ore-wrapper index e23e27c5a5..1021a08fd4 100755 --- a/src/cmd-ore-wrapper +++ b/src/cmd-ore-wrapper @@ -15,6 +15,7 @@ from cosalib.cli import ( BuildCli ) from cosalib.qemuvariants import get_qemu_variant +from cosalib.ibmcloud import get_ibmcloud_variant if __name__ == '__main__': log.basicConfig( @@ -96,7 +97,10 @@ Each target has its own sub options. To access them us: sys.exit() # Now _extend the parser with the cloud targets_ - build = get_qemu_variant(target, args) + if target in ['ibmcloud', 'powervs']: + build = get_ibmcloud_variant(target, args) + else: + build = get_qemu_variant(target, args) log.info(f"operating on {build.image_name}") if args.build_artifact: if args.force: diff --git a/src/cosalib/cli.py b/src/cosalib/cli.py index acf513dd69..67d5a04b45 100644 --- a/src/cosalib/cli.py +++ b/src/cosalib/cli.py @@ -12,7 +12,8 @@ digitalocean, gcp, vultr, - exoscale + exoscale, + ibmcloud ) CLOUD_CLI_TARGET = { @@ -37,6 +38,12 @@ "exoscale": (exoscale.exoscale_cli, exoscale.exoscale_run_ore, exoscale.exoscale_run_ore_replicate), + "ibmcloud": (ibmcloud.ibmcloud_cli, + ibmcloud.ibmcloud_run_ore, + ibmcloud.ibmcloud_run_ore_replicate), + "powervs": (ibmcloud.ibmcloud_cli, + ibmcloud.ibmcloud_run_ore, + ibmcloud.ibmcloud_run_ore_replicate), } diff --git a/src/cosalib/ibmcloud.py b/src/cosalib/ibmcloud.py new file mode 100644 index 0000000000..3101762421 --- /dev/null +++ b/src/cosalib/ibmcloud.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +# NOTE: PYTHONUNBUFFERED is set in cmdlib.sh for unbuffered output +# +# An operation that mutates a build by generating an ova +import logging as log +import urllib +import os.path +import sys +from cosalib.cmdlib import ( + run_verbose, + get_basearch +) +from tenacity import ( + retry, + stop_after_attempt +) + +cosa_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, f"{cosa_dir}/cosalib") +sys.path.insert(0, cosa_dir) + +from cosalib.qemuvariants import QemuVariantImage + + +OVA_TEMPLATE_FILE = '/usr/lib/coreos-assembler/powervs-template.xml' + +template_meta = """os-type = {os} +architecture = {basearch} +vol1-file = {image} +vol1-type = boot""" + +# Variant are OVA types that are derived from qemu images. +# To define new variants that use the QCOW2/raw disk image, simply, +# add its definition below: +VARIANTS = { + "ibmcloud": { + "image_format": "qcow2", + "platform": "ibmcloud", + "virtual_size": "100G", + }, + "powervs": { + "image_format": "raw", + "image_suffix": "ova", + "platform": "powervs", + "tar_members": [ + "disk.raw" + ] + }, +} + + +class IBMCloudImage(QemuVariantImage): + """ + PowerVSOVA's are based on the QemuVariant Image. This Class tries to do + the absolute bare minium, and is effectively a wrapper around the + QemuVariantImage Class. The only added functionality is the generation + of the OVF paramters. + """ + + def __init__(self, **kwargs): + variant = kwargs.pop("variant", "ibmcloud") + kwargs.update(VARIANTS.get(variant, {})) + QemuVariantImage.__init__(self, **kwargs) + # Set the QemuVariant mutate_callback so that OVA is called. + if variant == "powervs": + self.mutate_callback = self.write_ova + # Ensure that coreos.ovf is included in the tar + self.ovf_path = os.path.join(self._tmpdir, "coreos.ovf") + # Ensure that coreos.meta is included in the tar + self.meta_path = os.path.join(self._tmpdir, "coreos.meta") + + def generate_ovf_parameters(self, raw): + """ + Returns a dictionary with the parameters needed to create an OVF and meta file + based on the qemu, raw, and info from the build metadata + """ + image_size = os.stat(raw).st_size + image = f'{self.meta["name"]}-{self.meta["ostree-version"]}' + image_description = f'{self.meta["name"]} {self.meta["summary"]} {self.meta["ostree-version"]}' + + params = { + 'os': os.path.basename(raw).split("-")[0], + 'basearch': get_basearch(), + 'image_description': image_description, + 'image': image, + 'image_size': str(image_size), + } + + return params + + def write_ova(self, image_name): + """ + write_ova file. + + :param image_name: name of image to create OVF parameters for. + :type image_name: str + """ + ovf_params = self.generate_ovf_parameters(image_name) + + with open(OVA_TEMPLATE_FILE) as f: + template = f.read() + ovf_xml = template.format(**ovf_params) + + meta_text = template_meta.format(**ovf_params) + + with open(self.ovf_path, "w") as ovf: + ovf.write(ovf_xml) + + with open(self.meta_path, "w") as meta: + meta.write(meta_text) + + log.debug(ovf_xml) + # OVF descriptor must come first, then the manifest, then the meta file + self.tar_members.append(self.ovf_path) + self.tar_members.append(self.meta_path) + + +@retry(reraise=True, stop=stop_after_attempt(3)) +def ibmcloud_run_ore(build, args): + ore_args = ['ore'] + if args.log_level: + ore_args.extend(['--log-level', args.log_level]) + + if args.force: + ore_args.extend(['--force']) + + region = "us-east" + if args.region is not None and len(args.region) > 0: + region = args.region[0] + + platform = args.target + if args.cloud_object_storage is not None: + cloud_object_storage = args.cloud_object_storage + else: + cloud_object_storage = f"coreos-dev-image-{platform}" + + # powervs requires the image name to have an extension and also does not tolerate dots in the name. It affects the internal import from IBMCloud to the PowerVS systems + if platform == "powervs": + build_id = build.build_id.replace(".", "-") + ".ova" + else: + build_id = build.build_id + + ibmcloud_object_name = f"{build.build_name}-{build_id}" + ore_args.extend([ + 'ibmcloud', 'upload', + '--region', f"{region}", + '--cloud-object-storage', f"{cloud_object_storage}", + '--bucket', f"{args.bucket}", + '--name', ibmcloud_object_name, + '--file', f"{build.image_path}", + ]) + + run_verbose(ore_args) + url_path = urllib.parse.quote(( + f"s3.{region}.cloud-object-storage.appdomain.cloud/" + f"{args.bucket}/{ibmcloud_object_name}" + )) + + build.meta[platform] = { + 'image': ibmcloud_object_name, + 'bucket': args.bucket, + 'region': region, + 'url': f"https://{url_path}", + } + build.meta_write() # update build metadata + + +def ibmcloud_run_ore_replicate(build, args): + pass + + +def ibmcloud_cli(parser): + parser.add_argument("--bucket", help="S3 Bucket") + parser.add_argument("--cloud-object-storage", help="IBMCloud cloud object storage to upload to") + return parser + + +def get_ibmcloud_variant(variant, parser, kwargs={}): + """ + Helper function to get the IBMCloudImage Build Obj + """ + log.debug(f"returning IBMCloudImage for {variant}") + return IBMCloudImage( + buildroot=parser.buildroot, + build=parser.build, + schema=parser.schema, + variant=variant, + force=parser.force, + **kwargs) diff --git a/src/cosalib/qemuvariants.py b/src/cosalib/qemuvariants.py index f69ece8c18..9108c20ed9 100644 --- a/src/cosalib/qemuvariants.py +++ b/src/cosalib/qemuvariants.py @@ -90,11 +90,6 @@ "--format=oldgnu" ] }, - "ibmcloud": { - "image_format": "qcow2", - "platform": "ibmcloud", - "virtual_size": "100G", - }, "openstack": { "image_format": "qcow2", "platform": "openstack", diff --git a/src/powervs-template.xml b/src/powervs-template.xml new file mode 100644 index 0000000000..4661126fcb --- /dev/null +++ b/src/powervs-template.xml @@ -0,0 +1,38 @@ + + + + + + + Disk Section + + + + + {image} + + + + + + + + {image_description} + ppc64le + + + Storage resources + + Temporary clone for export + disk.raw + ovf:/disk/disk1 + 1 + 17 + True + + + + + {image} + + diff --git a/src/schema/v1.json b/src/schema/v1.json index 5c34a6c122..40e3199a80 100644 --- a/src/schema/v1.json +++ b/src/schema/v1.json @@ -72,6 +72,10 @@ "image", "url" ], + "optional": [ + "bucket", + "region" + ], "properties": { "image": { "$id":"#/cloudartifact/image", @@ -82,6 +86,16 @@ "$id":"#/cloudartifact/url", "type":"string", "title":"URL" + }, + "bucket": { + "$id":"#/cloudartifact/bucket", + "type":"string", + "title":"Bucket" + }, + "region": { + "$id":"#/cloudartifact/region", + "type":"string", + "title":"Region" } } }, @@ -182,6 +196,7 @@ "exoscale", "gcp", "ibmcloud", + "powervs", "images", "koji", "oscontainer", @@ -375,6 +390,7 @@ "exoscale", "gcp", "ibmcloud", + "powervs", "initramfs", "iso", "kernel", @@ -522,6 +538,12 @@ "title":"IBM Cloud", "$ref": "#/definitions/artifact" }, + "powervs": { + "$id":"#/properties/images/properties/powervs", + "type":"object", + "title":"Power Virtual Server", + "$ref": "#/definitions/artifact" + }, "gcp": { "$id":"#/properties/images/properties/gcp", "type":"object", @@ -792,6 +814,18 @@ } } }, + "ibmcloud": { + "$id":"#/properties/ibmcloud", + "type":"object", + "title":"IBM Cloud", + "$ref": "#/definitions/cloudartifact" + }, + "powervs": { + "$id":"#/properties/powervs", + "type":"object", + "title":"Power Virtual Server", + "$ref": "#/definitions/cloudartifact" + }, "release-payload": { "$id":"#/properties/release-payload", "type":"object",