support replication of images across regions for IBMCloud and PowerVS

mantle/cmd/ore/ibmcloud/copy-object.go

@@ -0,0 +1,78 @@
+// Copyright 2021 RedHat.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ibmcloud
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+)
+
+var (
+	cmdCopyObject = &cobra.Command{
+		Use:   "copy-object",
+		Short: "Copy IBMCloud object to a bucket",
+		Long:  "Copy an IBMCloud object from a bucket in a region to another bucket in a specified region",
+		Example: `  ore ibmcloud copy-object --source-name=image.qcow2 \
+		--destination-region=us-south \
+		--destination-bucket=coreos-dev-image-ibmcloud-us-south`,
+		RunE: runCopyObject,
+
+		SilenceUsage: true,
+	}
+
+	copyCloudObjectStorage string
+	sourceBucket           string
+	sourceName             string
+	destRegion             string
+	destBucket             string
+)
+
+func init() {
+	IbmCloud.AddCommand(cmdCopyObject)
+	cmdCopyObject.Flags().StringVar(&copyCloudObjectStorage, "cloud-object-storage", "coreos-dev-image-ibmcloud", "cloud object storage to be used")
+	cmdCopyObject.Flags().StringVar(&sourceBucket, "source-bucket", "coreos-dev-image-ibmcloud-us-east", "bucket where object needs to be copied from")
+	cmdCopyObject.Flags().StringVar(&sourceName, "source-name", "", "name of object to be copied")
+	cmdCopyObject.MarkFlagRequired("source-name")
+	cmdCopyObject.Flags().StringVar(&destRegion, "destination-region", "", "region to be copied to")
+	cmdCopyObject.MarkFlagRequired("destination-region")
+	cmdCopyObject.Flags().StringVar(&destBucket, "destination-bucket", "", "destination bucket to copy to")
+	cmdCopyObject.MarkFlagRequired("destination-bucket")
+}
+
+func runCopyObject(cmd *cobra.Command, args []string) error {
+	if err := API.NewS3Client(copyCloudObjectStorage, destRegion); err != nil {
+		return err
+	}
+
+	bucketExists, err := API.CheckBucketExists(destBucket)
+	if err != nil {
+		return err
+	}
+
+	if !bucketExists {
+		if err = API.CreateBucket(destBucket); err != nil {
+			return err
+		}
+	}
+
+	if err = API.CopyObject(sourceBucket, sourceName, destBucket); err != nil {
+		fmt.Fprintf(os.Stderr, "Couldn't copy objects: %v\n", err)
+		os.Exit(1)
+	}
+
+	return nil
+}

mantle/platform/api/ibmcloud/s3.go

@@ -20,10 +20,12 @@ package ibmcloud
 import (
 	"fmt"
 	"io"
+	"net/url"
 	"time"
 
 	"github.com/IBM-Cloud/bluemix-go/api/resource/resourcev2/controllerv2"
 	"github.com/IBM/ibm-cos-sdk-go/aws"
+	"github.com/IBM/ibm-cos-sdk-go/aws/awserr"
 	"github.com/IBM/ibm-cos-sdk-go/aws/credentials/ibmiam"
 	"github.com/IBM/ibm-cos-sdk-go/aws/session"
 	"github.com/IBM/ibm-cos-sdk-go/service/s3"
@@ -164,3 +166,28 @@ func (a *API) UploadObject(r io.Reader, objectName, bucketName string, force boo
 	plog.Infof("Upload completed successfully in %f seconds to location %s\n", time.Since(startTime).Seconds(), result.Location)
 	return err
 }
+
+// CopyObject - Copy an Object to a new location
+func (a *API) CopyObject(srcBucket, srcName, destBucket string) error {
+	_, err := a.s3client.s3Session.CopyObject(&s3.CopyObjectInput{
+		CopySource: aws.String(url.QueryEscape(fmt.Sprintf("%s/%s", srcBucket, srcName))),
+		Bucket:     aws.String(destBucket),
+		Key:        aws.String(srcName),
+	})
+	if err != nil {
+		if awserr, ok := err.(awserr.Error); ok {
+			if awserr.Code() == "BucketAlreadyOwnedByYou" {
+				return nil
+			}
+		}
+	}
+
+	// Wait to see if the item got copied
+	err = a.s3client.s3Session.WaitUntilObjectExists(&s3.HeadObjectInput{Bucket: aws.String(destBucket), Key: aws.String(srcName)})
+	if err != nil {
+		return fmt.Errorf("Error occurred while waiting for item %q to be copied to bucket %q, %v", srcName, destBucket, err)
+	}
+
+	plog.Infof("Item %q successfully copied from bucket %q to bucket %q\n", srcName, srcBucket, destBucket)
+	return err
+}

schema/cosa/cosa_v1.go

@@ -50,7 +50,7 @@ type Build struct {
 	FedoraCoreOsParentVersion string                `json:"fedora-coreos.parent-version,omitempty"`
 	Gcp                       *Gcp                  `json:"gcp,omitempty"`
 	GitDirty                  string                `json:"coreos-assembler.config-dirty,omitempty"`
-	IbmCloud                  *Cloudartifact        `json:"ibmcloud,omitempty"`
+	IbmCloud                  []Cloudartifact       `json:"ibmcloud,omitempty"`
 	ImageInputChecksum        string                `json:"coreos-assembler.image-input-checksum,omitempty"`
 	InputHasOfTheRpmOstree    string                `json:"rpm-ostree-inputhash"`
 	Koji                      *Koji                 `json:"koji,omitempty"`
@@ -70,7 +70,7 @@ type Build struct {
 	OverridesActive           bool                  `json:"coreos-assembler.overrides-active,omitempty"`
 	PkgdiffAgainstParent      PackageSetDifferences `json:"parent-pkgdiff,omitempty"`
 	PkgdiffBetweenBuilds      PackageSetDifferences `json:"pkgdiff,omitempty"`
-	PowerVirtualServer        *Cloudartifact        `json:"powervs,omitempty"`
+	PowerVirtualServer        []Cloudartifact       `json:"powervs,omitempty"`
 	ReleasePayload            *Image                `json:"release-payload,omitempty"`
 }
 
@@ -103,7 +103,8 @@ type BuildArtifacts struct {
 
 type Cloudartifact struct {
 	Bucket string `json:"bucket,omitempty"`
-	Image  string `json:"image"`
+	Image  string `json:"image,omitempty"`
+	Object string `json:"object,omitempty"`
 	Region string `json:"region,omitempty"`
 	URL    string `json:"url"`
 }

schema/cosa/schema_doc.go

@@ -74,10 +74,11 @@ var generatedSchemaJSON = `{
       "cloudartifact": {
          "type": "object",
          "required": [
-             "image",
              "url"
          ],
          "optional": [
+             "image",
+             "object",
              "bucket",
              "region"
          ],
@@ -101,6 +102,11 @@ var generatedSchemaJSON = `{
              "$id":"#/cloudartifact/region",
              "type":"string",
              "title":"Region"
+            },
+            "object": {
+             "$id":"#/cloudartifact/object",
+             "type":"string",
+             "title":"Object"
             }
           }
      },
@@ -178,7 +184,7 @@ var generatedSchemaJSON = `{
       }
  },
  "$schema":"http://json-schema.org/draft-07/schema#",
- "$id":"https://github.com/coreos/coreos-assembler/blob/main/schema/v1.json",
+ "$id":"https://github.com/coreos/coreos-assembler/blob/main/v1.json.json",
  "type":"object",
  "title":"CoreOS Assember v1 meta.json schema",
  "required": [
@@ -820,16 +826,22 @@ var generatedSchemaJSON = `{
       }
     },
    "ibmcloud": {
-      "$id":"#/properties/ibmcloud",
-      "type":"object",
-      "title":"IBM Cloud",
-      "$ref": "#/definitions/cloudartifact"
+     "$id":"#/properties/ibmcloud",
+     "type":"array",
+     "title":"IBM Cloud",
+     "items": {
+       "type":"object",
+       "$ref": "#/definitions/cloudartifact"
+      }
     },
    "powervs": {
-      "$id":"#/properties/powervs",
-      "type":"object",
-      "title":"Power Virtual Server",
-      "$ref": "#/definitions/cloudartifact"
+     "$id":"#/properties/powervs",
+     "type":"array",
+     "title":"Power Virtual Server",
+     "items": {
+       "type":"object",
+       "$ref": "#/definitions/cloudartifact"
+      }
     },
     "release-payload": {
       "$id":"#/properties/release-payload",