From 1bf9fd3df78484d7643df2d49118591ace9a0165 Mon Sep 17 00:00:00 2001 From: Prashanth Sundararaman Date: Wed, 11 Aug 2021 15:48:56 -0700 Subject: [PATCH 1/4] ore/ibmcloud: add copy-object command in order to replicate images from a bucket in a particular region to another bucket in a different region, introduce the copy-object command. this command can only copy to buckets in a particular region because for IBMCloud, the s3 client needs a specfic endpoint and thus needs to be associated with a region. --- mantle/cmd/ore/ibmcloud/copy-object.go | 78 ++++++++++++++++++++++++++ mantle/platform/api/ibmcloud/s3.go | 27 +++++++++ 2 files changed, 105 insertions(+) create mode 100644 mantle/cmd/ore/ibmcloud/copy-object.go diff --git a/mantle/cmd/ore/ibmcloud/copy-object.go b/mantle/cmd/ore/ibmcloud/copy-object.go new file mode 100644 index 0000000000..a188ab0a33 --- /dev/null +++ b/mantle/cmd/ore/ibmcloud/copy-object.go @@ -0,0 +1,78 @@ +// Copyright 2021 RedHat. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ibmcloud + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" +) + +var ( + cmdCopyObject = &cobra.Command{ + Use: "copy-object", + Short: "Copy IBMCloud object to a bucket", + Long: "Copy an IBMCloud object from a bucket in a region to another bucket in a specified region", + Example: ` ore ibmcloud copy-object --source-name=image.qcow2 \ + --destination-region=us-south \ + --destination-bucket=coreos-dev-image-ibmcloud-us-south`, + RunE: runCopyObject, + + SilenceUsage: true, + } + + copyCloudObjectStorage string + sourceBucket string + sourceName string + destRegion string + destBucket string +) + +func init() { + IbmCloud.AddCommand(cmdCopyObject) + cmdCopyObject.Flags().StringVar(©CloudObjectStorage, "cloud-object-storage", "coreos-dev-image-ibmcloud", "cloud object storage to be used") + cmdCopyObject.Flags().StringVar(&sourceBucket, "source-bucket", "coreos-dev-image-ibmcloud-us-east", "bucket where object needs to be copied from") + cmdCopyObject.Flags().StringVar(&sourceName, "source-name", "", "name of object to be copied") + cmdCopyObject.MarkFlagRequired("source-name") + cmdCopyObject.Flags().StringVar(&destRegion, "destination-region", "", "region to be copied to") + cmdCopyObject.MarkFlagRequired("destination-region") + cmdCopyObject.Flags().StringVar(&destBucket, "destination-bucket", "", "destination bucket to copy to") + cmdCopyObject.MarkFlagRequired("destination-bucket") +} + +func runCopyObject(cmd *cobra.Command, args []string) error { + if err := API.NewS3Client(copyCloudObjectStorage, destRegion); err != nil { + return err + } + + bucketExists, err := API.CheckBucketExists(destBucket) + if err != nil { + return err + } + + if !bucketExists { + if err = API.CreateBucket(destBucket); err != nil { + return err + } + } + + if err = API.CopyObject(sourceBucket, sourceName, destBucket); err != nil { + fmt.Fprintf(os.Stderr, "Couldn't copy objects: %v\n", err) + os.Exit(1) + } + + return nil +} diff --git a/mantle/platform/api/ibmcloud/s3.go b/mantle/platform/api/ibmcloud/s3.go index 5f4ee9aa36..521a14a551 100644 --- a/mantle/platform/api/ibmcloud/s3.go +++ b/mantle/platform/api/ibmcloud/s3.go @@ -20,10 +20,12 @@ package ibmcloud import ( "fmt" "io" + "net/url" "time" "github.com/IBM-Cloud/bluemix-go/api/resource/resourcev2/controllerv2" "github.com/IBM/ibm-cos-sdk-go/aws" + "github.com/IBM/ibm-cos-sdk-go/aws/awserr" "github.com/IBM/ibm-cos-sdk-go/aws/credentials/ibmiam" "github.com/IBM/ibm-cos-sdk-go/aws/session" "github.com/IBM/ibm-cos-sdk-go/service/s3" @@ -164,3 +166,28 @@ func (a *API) UploadObject(r io.Reader, objectName, bucketName string, force boo plog.Infof("Upload completed successfully in %f seconds to location %s\n", time.Since(startTime).Seconds(), result.Location) return err } + +// CopyObject - Copy an Object to a new location +func (a *API) CopyObject(srcBucket, srcName, destBucket string) error { + _, err := a.s3client.s3Session.CopyObject(&s3.CopyObjectInput{ + CopySource: aws.String(url.QueryEscape(fmt.Sprintf("%s/%s", srcBucket, srcName))), + Bucket: aws.String(destBucket), + Key: aws.String(srcName), + }) + if err != nil { + if awserr, ok := err.(awserr.Error); ok { + if awserr.Code() == "BucketAlreadyOwnedByYou" { + return nil + } + } + } + + // Wait to see if the item got copied + err = a.s3client.s3Session.WaitUntilObjectExists(&s3.HeadObjectInput{Bucket: aws.String(destBucket), Key: aws.String(srcName)}) + if err != nil { + return fmt.Errorf("Error occurred while waiting for item %q to be copied to bucket %q, %v", srcName, destBucket, err) + } + + plog.Infof("Item %q successfully copied from bucket %q to bucket %q\n", srcName, srcBucket, destBucket) + return err +} From 4cb042554e413e8870a66b030e05780abfbe5317 Mon Sep 17 00:00:00 2001 From: Prashanth Sundararaman Date: Wed, 11 Aug 2021 15:53:23 -0700 Subject: [PATCH 2/4] schema: change ibmcloud and powervs artifacts from object to array since we want to support replication across regional buckets, change schema to an array which contains information on image name, region, bucket and url --- .../coreos-assembler-schema/cosa/cosa_v1.go | 7 ++-- .../cosa/schema_doc.go | 32 +++++++++++++------ .../coreos-assembler-schema/cosa/cosa_v1.go | 7 ++-- .../cosa/schema_doc.go | 32 +++++++++++++------ schema/cosa/cosa_v1.go | 7 ++-- schema/cosa/schema_doc.go | 32 +++++++++++++------ schema/generate-schema.sh | 2 +- schema/v1.json | 30 +++++++++++------ src/v1.json | 30 +++++++++++------ 9 files changed, 121 insertions(+), 58 deletions(-) diff --git a/gangplank/vendor/github.com/coreos/coreos-assembler-schema/cosa/cosa_v1.go b/gangplank/vendor/github.com/coreos/coreos-assembler-schema/cosa/cosa_v1.go index fb1595171f..4f9df58e6d 100644 --- a/gangplank/vendor/github.com/coreos/coreos-assembler-schema/cosa/cosa_v1.go +++ b/gangplank/vendor/github.com/coreos/coreos-assembler-schema/cosa/cosa_v1.go @@ -50,7 +50,7 @@ type Build struct { FedoraCoreOsParentVersion string `json:"fedora-coreos.parent-version,omitempty"` Gcp *Gcp `json:"gcp,omitempty"` GitDirty string `json:"coreos-assembler.config-dirty,omitempty"` - IbmCloud *Cloudartifact `json:"ibmcloud,omitempty"` + IbmCloud []Cloudartifact `json:"ibmcloud,omitempty"` ImageInputChecksum string `json:"coreos-assembler.image-input-checksum,omitempty"` InputHasOfTheRpmOstree string `json:"rpm-ostree-inputhash"` Koji *Koji `json:"koji,omitempty"` @@ -70,7 +70,7 @@ type Build struct { OverridesActive bool `json:"coreos-assembler.overrides-active,omitempty"` PkgdiffAgainstParent PackageSetDifferences `json:"parent-pkgdiff,omitempty"` PkgdiffBetweenBuilds PackageSetDifferences `json:"pkgdiff,omitempty"` - PowerVirtualServer *Cloudartifact `json:"powervs,omitempty"` + PowerVirtualServer []Cloudartifact `json:"powervs,omitempty"` ReleasePayload *Image `json:"release-payload,omitempty"` } @@ -103,7 +103,8 @@ type BuildArtifacts struct { type Cloudartifact struct { Bucket string `json:"bucket,omitempty"` - Image string `json:"image"` + Image string `json:"image,omitempty"` + Object string `json:"object,omitempty"` Region string `json:"region,omitempty"` URL string `json:"url"` } diff --git a/gangplank/vendor/github.com/coreos/coreos-assembler-schema/cosa/schema_doc.go b/gangplank/vendor/github.com/coreos/coreos-assembler-schema/cosa/schema_doc.go index 2218e67c24..7438eeaf44 100644 --- a/gangplank/vendor/github.com/coreos/coreos-assembler-schema/cosa/schema_doc.go +++ b/gangplank/vendor/github.com/coreos/coreos-assembler-schema/cosa/schema_doc.go @@ -74,10 +74,11 @@ var generatedSchemaJSON = `{ "cloudartifact": { "type": "object", "required": [ - "image", "url" ], "optional": [ + "image", + "object", "bucket", "region" ], @@ -101,6 +102,11 @@ var generatedSchemaJSON = `{ "$id":"#/cloudartifact/region", "type":"string", "title":"Region" + }, + "object": { + "$id":"#/cloudartifact/object", + "type":"string", + "title":"Object" } } }, @@ -178,7 +184,7 @@ var generatedSchemaJSON = `{ } }, "$schema":"http://json-schema.org/draft-07/schema#", - "$id":"http://github.com/coreos/coreos-assembler/blob/main/schema/v1.json", + "$id":"http://github.com/coreos/coreos-assembler/blob/main/v1.json.json", "type":"object", "title":"CoreOS Assember v1 meta.json schema", "required": [ @@ -820,16 +826,22 @@ var generatedSchemaJSON = `{ } }, "ibmcloud": { - "$id":"#/properties/ibmcloud", - "type":"object", - "title":"IBM Cloud", - "$ref": "#/definitions/cloudartifact" + "$id":"#/properties/ibmcloud", + "type":"array", + "title":"IBM Cloud", + "items": { + "type":"object", + "$ref": "#/definitions/cloudartifact" + } }, "powervs": { - "$id":"#/properties/powervs", - "type":"object", - "title":"Power Virtual Server", - "$ref": "#/definitions/cloudartifact" + "$id":"#/properties/powervs", + "type":"array", + "title":"Power Virtual Server", + "items": { + "type":"object", + "$ref": "#/definitions/cloudartifact" + } }, "release-payload": { "$id":"#/properties/release-payload", diff --git a/mantle/vendor/github.com/coreos/coreos-assembler-schema/cosa/cosa_v1.go b/mantle/vendor/github.com/coreos/coreos-assembler-schema/cosa/cosa_v1.go index fb1595171f..4f9df58e6d 100644 --- a/mantle/vendor/github.com/coreos/coreos-assembler-schema/cosa/cosa_v1.go +++ b/mantle/vendor/github.com/coreos/coreos-assembler-schema/cosa/cosa_v1.go @@ -50,7 +50,7 @@ type Build struct { FedoraCoreOsParentVersion string `json:"fedora-coreos.parent-version,omitempty"` Gcp *Gcp `json:"gcp,omitempty"` GitDirty string `json:"coreos-assembler.config-dirty,omitempty"` - IbmCloud *Cloudartifact `json:"ibmcloud,omitempty"` + IbmCloud []Cloudartifact `json:"ibmcloud,omitempty"` ImageInputChecksum string `json:"coreos-assembler.image-input-checksum,omitempty"` InputHasOfTheRpmOstree string `json:"rpm-ostree-inputhash"` Koji *Koji `json:"koji,omitempty"` @@ -70,7 +70,7 @@ type Build struct { OverridesActive bool `json:"coreos-assembler.overrides-active,omitempty"` PkgdiffAgainstParent PackageSetDifferences `json:"parent-pkgdiff,omitempty"` PkgdiffBetweenBuilds PackageSetDifferences `json:"pkgdiff,omitempty"` - PowerVirtualServer *Cloudartifact `json:"powervs,omitempty"` + PowerVirtualServer []Cloudartifact `json:"powervs,omitempty"` ReleasePayload *Image `json:"release-payload,omitempty"` } @@ -103,7 +103,8 @@ type BuildArtifacts struct { type Cloudartifact struct { Bucket string `json:"bucket,omitempty"` - Image string `json:"image"` + Image string `json:"image,omitempty"` + Object string `json:"object,omitempty"` Region string `json:"region,omitempty"` URL string `json:"url"` } diff --git a/mantle/vendor/github.com/coreos/coreos-assembler-schema/cosa/schema_doc.go b/mantle/vendor/github.com/coreos/coreos-assembler-schema/cosa/schema_doc.go index 2218e67c24..7438eeaf44 100644 --- a/mantle/vendor/github.com/coreos/coreos-assembler-schema/cosa/schema_doc.go +++ b/mantle/vendor/github.com/coreos/coreos-assembler-schema/cosa/schema_doc.go @@ -74,10 +74,11 @@ var generatedSchemaJSON = `{ "cloudartifact": { "type": "object", "required": [ - "image", "url" ], "optional": [ + "image", + "object", "bucket", "region" ], @@ -101,6 +102,11 @@ var generatedSchemaJSON = `{ "$id":"#/cloudartifact/region", "type":"string", "title":"Region" + }, + "object": { + "$id":"#/cloudartifact/object", + "type":"string", + "title":"Object" } } }, @@ -178,7 +184,7 @@ var generatedSchemaJSON = `{ } }, "$schema":"http://json-schema.org/draft-07/schema#", - "$id":"http://github.com/coreos/coreos-assembler/blob/main/schema/v1.json", + "$id":"http://github.com/coreos/coreos-assembler/blob/main/v1.json.json", "type":"object", "title":"CoreOS Assember v1 meta.json schema", "required": [ @@ -820,16 +826,22 @@ var generatedSchemaJSON = `{ } }, "ibmcloud": { - "$id":"#/properties/ibmcloud", - "type":"object", - "title":"IBM Cloud", - "$ref": "#/definitions/cloudartifact" + "$id":"#/properties/ibmcloud", + "type":"array", + "title":"IBM Cloud", + "items": { + "type":"object", + "$ref": "#/definitions/cloudartifact" + } }, "powervs": { - "$id":"#/properties/powervs", - "type":"object", - "title":"Power Virtual Server", - "$ref": "#/definitions/cloudartifact" + "$id":"#/properties/powervs", + "type":"array", + "title":"Power Virtual Server", + "items": { + "type":"object", + "$ref": "#/definitions/cloudartifact" + } }, "release-payload": { "$id":"#/properties/release-payload", diff --git a/schema/cosa/cosa_v1.go b/schema/cosa/cosa_v1.go index fb1595171f..4f9df58e6d 100644 --- a/schema/cosa/cosa_v1.go +++ b/schema/cosa/cosa_v1.go @@ -50,7 +50,7 @@ type Build struct { FedoraCoreOsParentVersion string `json:"fedora-coreos.parent-version,omitempty"` Gcp *Gcp `json:"gcp,omitempty"` GitDirty string `json:"coreos-assembler.config-dirty,omitempty"` - IbmCloud *Cloudartifact `json:"ibmcloud,omitempty"` + IbmCloud []Cloudartifact `json:"ibmcloud,omitempty"` ImageInputChecksum string `json:"coreos-assembler.image-input-checksum,omitempty"` InputHasOfTheRpmOstree string `json:"rpm-ostree-inputhash"` Koji *Koji `json:"koji,omitempty"` @@ -70,7 +70,7 @@ type Build struct { OverridesActive bool `json:"coreos-assembler.overrides-active,omitempty"` PkgdiffAgainstParent PackageSetDifferences `json:"parent-pkgdiff,omitempty"` PkgdiffBetweenBuilds PackageSetDifferences `json:"pkgdiff,omitempty"` - PowerVirtualServer *Cloudartifact `json:"powervs,omitempty"` + PowerVirtualServer []Cloudartifact `json:"powervs,omitempty"` ReleasePayload *Image `json:"release-payload,omitempty"` } @@ -103,7 +103,8 @@ type BuildArtifacts struct { type Cloudartifact struct { Bucket string `json:"bucket,omitempty"` - Image string `json:"image"` + Image string `json:"image,omitempty"` + Object string `json:"object,omitempty"` Region string `json:"region,omitempty"` URL string `json:"url"` } diff --git a/schema/cosa/schema_doc.go b/schema/cosa/schema_doc.go index 2218e67c24..7438eeaf44 100644 --- a/schema/cosa/schema_doc.go +++ b/schema/cosa/schema_doc.go @@ -74,10 +74,11 @@ var generatedSchemaJSON = `{ "cloudartifact": { "type": "object", "required": [ - "image", "url" ], "optional": [ + "image", + "object", "bucket", "region" ], @@ -101,6 +102,11 @@ var generatedSchemaJSON = `{ "$id":"#/cloudartifact/region", "type":"string", "title":"Region" + }, + "object": { + "$id":"#/cloudartifact/object", + "type":"string", + "title":"Object" } } }, @@ -178,7 +184,7 @@ var generatedSchemaJSON = `{ } }, "$schema":"http://json-schema.org/draft-07/schema#", - "$id":"http://github.com/coreos/coreos-assembler/blob/main/schema/v1.json", + "$id":"http://github.com/coreos/coreos-assembler/blob/main/v1.json.json", "type":"object", "title":"CoreOS Assember v1 meta.json schema", "required": [ @@ -820,16 +826,22 @@ var generatedSchemaJSON = `{ } }, "ibmcloud": { - "$id":"#/properties/ibmcloud", - "type":"object", - "title":"IBM Cloud", - "$ref": "#/definitions/cloudartifact" + "$id":"#/properties/ibmcloud", + "type":"array", + "title":"IBM Cloud", + "items": { + "type":"object", + "$ref": "#/definitions/cloudartifact" + } }, "powervs": { - "$id":"#/properties/powervs", - "type":"object", - "title":"Power Virtual Server", - "$ref": "#/definitions/cloudartifact" + "$id":"#/properties/powervs", + "type":"array", + "title":"Power Virtual Server", + "items": { + "type":"object", + "$ref": "#/definitions/cloudartifact" + } }, "release-payload": { "$id":"#/properties/release-payload", diff --git a/schema/generate-schema.sh b/schema/generate-schema.sh index 88cc4b33b2..e70baed327 100755 --- a/schema/generate-schema.sh +++ b/schema/generate-schema.sh @@ -10,7 +10,7 @@ if [ ! -x "${GOBIN}/schematyper" ]; then fi schema_version="v1" -schema_json="../src/schema/${schema_version}.json" +schema_json="../schema/${schema_version}.json" echo "Generating COSA Schema ${schema_version}" out="${tdir}/cosa_${schema_version}.go" diff --git a/schema/v1.json b/schema/v1.json index 9295b245d3..889129bc81 100644 --- a/schema/v1.json +++ b/schema/v1.json @@ -69,10 +69,11 @@ "cloudartifact": { "type": "object", "required": [ - "image", "url" ], "optional": [ + "image", + "object", "bucket", "region" ], @@ -96,6 +97,11 @@ "$id":"#/cloudartifact/region", "type":"string", "title":"Region" + }, + "object": { + "$id":"#/cloudartifact/object", + "type":"string", + "title":"Object" } } }, @@ -815,16 +821,22 @@ } }, "ibmcloud": { - "$id":"#/properties/ibmcloud", - "type":"object", - "title":"IBM Cloud", - "$ref": "#/definitions/cloudartifact" + "$id":"#/properties/ibmcloud", + "type":"array", + "title":"IBM Cloud", + "items": { + "type":"object", + "$ref": "#/definitions/cloudartifact" + } }, "powervs": { - "$id":"#/properties/powervs", - "type":"object", - "title":"Power Virtual Server", - "$ref": "#/definitions/cloudartifact" + "$id":"#/properties/powervs", + "type":"array", + "title":"Power Virtual Server", + "items": { + "type":"object", + "$ref": "#/definitions/cloudartifact" + } }, "release-payload": { "$id":"#/properties/release-payload", diff --git a/src/v1.json b/src/v1.json index 9295b245d3..889129bc81 100644 --- a/src/v1.json +++ b/src/v1.json @@ -69,10 +69,11 @@ "cloudartifact": { "type": "object", "required": [ - "image", "url" ], "optional": [ + "image", + "object", "bucket", "region" ], @@ -96,6 +97,11 @@ "$id":"#/cloudartifact/region", "type":"string", "title":"Region" + }, + "object": { + "$id":"#/cloudartifact/object", + "type":"string", + "title":"Object" } } }, @@ -815,16 +821,22 @@ } }, "ibmcloud": { - "$id":"#/properties/ibmcloud", - "type":"object", - "title":"IBM Cloud", - "$ref": "#/definitions/cloudartifact" + "$id":"#/properties/ibmcloud", + "type":"array", + "title":"IBM Cloud", + "items": { + "type":"object", + "$ref": "#/definitions/cloudartifact" + } }, "powervs": { - "$id":"#/properties/powervs", - "type":"object", - "title":"Power Virtual Server", - "$ref": "#/definitions/cloudartifact" + "$id":"#/properties/powervs", + "type":"array", + "title":"Power Virtual Server", + "items": { + "type":"object", + "$ref": "#/definitions/cloudartifact" + } }, "release-payload": { "$id":"#/properties/release-payload", From 360b81a80e176fcdd0b497ec524b01465e1147b9 Mon Sep 17 00:00:00 2001 From: Prashanth Sundararaman Date: Wed, 11 Aug 2021 15:55:00 -0700 Subject: [PATCH 3/4] cosalib/ibmcloud.py: support replication of images across regions one of the requirements for the PowerVS platform is to have ova images replicated across collocation zones so clusters can be deployed in those zones. This change allows images to be replicated across specific regions and in buckets that are specified by prefixes so that it is helpful for the pipeline to create buckets and copy these images to those buckets. --- src/cosalib/ibmcloud.py | 105 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 97 insertions(+), 8 deletions(-) diff --git a/src/cosalib/ibmcloud.py b/src/cosalib/ibmcloud.py index 43d448b6b4..bab2cb9e11 100644 --- a/src/cosalib/ibmcloud.py +++ b/src/cosalib/ibmcloud.py @@ -2,6 +2,7 @@ # NOTE: PYTHONUNBUFFERED is set in cmdlib.sh for unbuffered output # # An operation that mutates a build by generating an ova +import subprocess import logging as log import urllib import os.path @@ -131,13 +132,13 @@ def ibmcloud_run_ore(build, args): else: cloud_object_storage = f"coreos-dev-image-{platform}" - # powervs requires the image name to have an extension and also does not tolerate dots in the name. It affects the internal import from IBMCloud to the PowerVS systems + ibmcloud_object_name = f"{build.build_name}-{build.build_id}-{build.basearch}-{build.platform}" if platform == "powervs": - build_id = build.build_id.replace(".", "-") + ".ova" - else: - build_id = build.build_id + # powervs requires the image name to have an extension and also does not + # tolerate dots in the name. It affects the internal import from IBMCloud + # to the PowerVS systems + ibmcloud_object_name = ibmcloud_object_name.replace(".", "-") + ".ova" - ibmcloud_object_name = f"{build.build_name}-{build_id}" ore_args.extend([ 'ibmcloud', 'upload', '--region', f"{region}", @@ -159,21 +160,109 @@ def ibmcloud_run_ore(build, args): f"{args.bucket}/{ibmcloud_object_name}" )) - build.meta[platform] = { - 'image': ibmcloud_object_name, + build.meta[platform] = [{ + 'object': ibmcloud_object_name, 'bucket': args.bucket, 'region': region, 'url': f"https://{url_path}", - } + }] build.meta_write() # update build metadata def ibmcloud_run_ore_replicate(build, args): + build.refresh_meta() + platform = args.target + if platform == "powervs": + ibmcloud_img_data = build.meta.get('powervs', []) + else: + ibmcloud_img_data = build.meta.get('ibmcloud', []) + if len(ibmcloud_img_data) < 1: + raise SystemExit(("buildmeta doesn't contain source images. " + "Run buildextend-{platform} first")) + + # define regions - https://cloud.ibm.com/docs/power-iaas?topic=power-iaas-creating-power-virtual-server#creating-service + # PowerVS insatnces are supported in all the regions where cloud object storage can be created. This list is common for + # both IBMCloud and PowerVS. + if not args.region: + args.region = ['au-syd', 'br-sao', 'ca-tor', 'eu-de', 'eu-gb', 'jp-osa', 'jp-tok', 'us-east', 'us-south'] + log.info(("default: replicating to all regions. If this is not " + " desirable, use '--regions'")) + + log.info("replicating to regions: %s", args.region) + + # only replicate to regions that don't already exist + existing_regions = [item['region'] for item in ibmcloud_img_data] + duplicates = list(set(args.region).intersection(existing_regions)) + if len(duplicates) > 0: + print((f"Images already exist in {duplicates} region(s)" + ", skipping listed region(s)...")) + region_list = list(set(args.region) - set(duplicates)) + if len(region_list) == 0: + print("no new regions detected") + sys.exit(0) + + source_object = ibmcloud_img_data[0]['object'] + source_bucket = ibmcloud_img_data[0]['bucket'] + + if args.cloud_object_storage is not None: + cloud_object_storage = args.cloud_object_storage + else: + cloud_object_storage = f"coreos-dev-image-{platform}" + + if args.bucket_prefix is not None: + bucket_prefix = args.bucket_prefix + else: + bucket_prefix = f"coreos-dev-image-{platform}" + + ore_args = [ + 'ore', + '--log-level', args.log_level, + 'ibmcloud', 'copy-object', + '--cloud-object-storage', cloud_object_storage, + '--source-name', source_object, + '--source-bucket', source_bucket + ] + + if args.credentials_file is not None: + ore_args.extend(['--credentials-file', f"{args.credentials_file}"]) + + upload_failed_in_region = None + + for upload_region in region_list: + region_ore_args = ore_args.copy() + ['--destination-region', upload_region, + '--destination-bucket', f"{bucket_prefix}-{upload_region}"] + print("+ {}".format(subprocess.list2cmdline(region_ore_args))) + try: + subprocess.check_output(region_ore_args) + except subprocess.CalledProcessError: + upload_failed_in_region = upload_region + break + + url_path = urllib.parse.quote(( + f"s3.{upload_region}.cloud-object-storage.appdomain.cloud/" + f"{bucket_prefix}-{upload_region}/{source_object}" + )) + + ibmcloud_img_data.extend([ + { + 'object': source_object, + 'bucket': f"{bucket_prefix}-{upload_region}", + 'region': upload_region, + 'url': f"https://{url_path}" + } + ]) + + build.meta[platform] = ibmcloud_img_data + build.meta_write() + + if upload_failed_in_region is not None: + raise Exception(f"Upload failed in {upload_failed_in_region} region") pass def ibmcloud_cli(parser): parser.add_argument("--bucket", help="S3 Bucket") + parser.add_argument("--bucket-prefix", help="S3 Bucket prefix to replicate across regional buckets") parser.add_argument("--cloud-object-storage", help="IBMCloud cloud object storage to upload to") parser.add_argument("--credentials-file", help="Path to IBMCloud auth file") return parser From bb30d84d7d5aa2169c2016faa8348cbe68673f21 Mon Sep 17 00:00:00 2001 From: Prashanth Sundararaman Date: Tue, 7 Sep 2021 11:58:54 -0700 Subject: [PATCH 4/4] generate-release-meta: add support for uploaded IBMCloud and PowerVS images we now have the ability to replicate IBMCloud/PowerVS images. add them to the release.json --- src/cmd-generate-release-meta | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/cmd-generate-release-meta b/src/cmd-generate-release-meta index 1dde475440..cb14a493dc 100755 --- a/src/cmd-generate-release-meta +++ b/src/cmd-generate-release-meta @@ -124,7 +124,7 @@ def append_build(out, input_): # build the architectures dict arch_dict = {"media": {}} ensure_dup(input_, arch_dict, "ostree-commit", "commit") - platforms = ["aliyun", "aws", "azure", "azurestack", "digitalocean", "exoscale", "gcp", "ibmcloud", "metal", "openstack", "qemu", "vmware", "vultr"] + platforms = ["aliyun", "aws", "azure", "azurestack", "digitalocean", "exoscale", "gcp", "ibmcloud", "metal", "openstack", "powervs", "qemu", "vmware", "vultr"] for platform in platforms: if input_.get("images", {}).get(platform, None) is not None: print(f" - {platform}") @@ -146,6 +146,19 @@ def append_build(out, input_): "image": cloud_dict[image_field] } + # IBMCloud/PowerVS specific additions + for meta_key, cloud, object_field, bucket_field, url_field in \ + ("ibmcloud", "ibmcloud", "object", "bucket", "url"), \ + ("powervs", "powervs", "object", "bucket", "url"): + if input_.get(meta_key, None) is not None: + arch_dict["media"].setdefault(cloud, {}).setdefault("images", {}) + for cloud_dict in input_.get(meta_key): + arch_dict["media"][cloud]["images"][cloud_dict["region"]] = { + "object": cloud_dict[object_field], + "bucket": cloud_dict[bucket_field], + "url": cloud_dict[url_field] + } + # GCP specific additions if input_.get("gcp", None) is not None: arch_dict["media"].setdefault("gcp", {}).setdefault("image", {})