Switch to AWS CI/CD

prow_config.yaml

@@ -1,98 +1,98 @@
 # This file configures the workflows to trigger in our Prow jobs.
 # see kubeflow/testing/py/run_e2e_workflow.py
 workflows:
-  - app_dir: kubeflow/katib/test/workflows
-    component: workflows-v1alpha3
-    name: e2e-v1alpha3
-    job_types:
-      - presubmit
-    include_dirs:
-      - pkg/apis/controller/common/v1alpha3/*
-      - pkg/apis/controller/experiments/v1alpha3/*
-      - pkg/apis/controller/trials/v1alpha3/*
-      - pkg/apis/controller/suggestions/v1alpha3/*
-      - pkg/apis/controller/a*.go
-      - pkg/apis/manager/health/*
-      - pkg/apis/manager/v1alpha3/*
-      - pkg/apis/v1alpha3/*
-      - pkg/common/v1alpha3/*
-      - pkg/controller.v1alpha3/*
-      - pkg/db/v1alpha3/*
-      - pkg/job/v1alpha3/*
-      - pkg/metricscollector/v1alpha3/*
-      - pkg/suggestion/v1alpha3/*
-      - pkg/ui/v1alpha3/*
-      - pkg/util/v1alpha3/*
-      - pkg/webhook/v1alpha3/*
-      - cmd/db-manager/v1alpha3/*
-      - cmd/katib-controller/v1alpha3/*
-      - cmd/metricscollector/v1alpha3/*
-      - cmd/suggestion/chocolate/v1alpha3/*
-      - cmd/suggestion/hyperband/v1alpha3/*
-      - cmd/suggestion/hyperopt/v1alpha3/*
-      - cmd/suggestion/nas/enas/v1alpha3/*
-      - cmd/suggestion/nas/darts/v1alpha3/*
-      - cmd/suggestion/skopt/v1alpha3/*
-      - cmd/suggestion/goptuna/v1alpha3/*
-      - cmd/ui/v1alpha3/*
-      - examples/v1alpha3/*
-      - test/e2e/v1alpha3/*
-      - test/scripts/v1alpha3/*
-      - test/suggestion/v1alpha3/*
-      - test/unit/v1alpha3/*
-      - test/workflows/*
-      - manifests/v1alpha3/*
-      - scripts/v1alpha3/*
-      - vendor/*
-      - prow_config.yaml
-    params:
-      registry: "gcr.io/automl-ci"
-  - app_dir: kubeflow/katib/test/workflows
-    component: workflows-v1alpha3
-    name: e2e-v1alpha3
-    job_types:
-      - postsubmit
-    include_dirs:
-      - pkg/apis/controller/common/v1alpha3/*
-      - pkg/apis/controller/experiments/v1alpha3/*
-      - pkg/apis/controller/trials/v1alpha3/*
-      - pkg/apis/controller/suggestions/v1alpha3/*
-      - pkg/apis/controller/a*.go
-      - pkg/apis/manager/health/*
-      - pkg/apis/manager/v1alpha3/*
-      - pkg/apis/v1alpha3/*
-      - pkg/common/v1alpha3/*
-      - pkg/controller.v1alpha3/*
-      - pkg/db/v1alpha3/*
-      - pkg/job/v1alpha3/*
-      - pkg/metricscollector/v1alpha3/*
-      - pkg/suggestion/v1alpha3/*
-      - pkg/ui/v1alpha3/*
-      - pkg/util/v1alpha3/*
-      - pkg/webhook/v1alpha3/*
-      - cmd/db-manager/v1alpha3/*
-      - cmd/katib-controller/v1alpha3/*
-      - cmd/metricscollector/v1alpha3/*
-      - cmd/suggestion/chocolate/v1alpha3/*
-      - cmd/suggestion/hyperband/v1alpha3/*
-      - cmd/suggestion/hyperopt/v1alpha3/*
-      - cmd/suggestion/nas/enas/v1alpha3/*
-      - cmd/suggestion/nas/darts/v1alpha3/*
-      - cmd/suggestion/skopt/v1alpha3/*
-      - cmd/suggestion/goptuna/v1alpha3/*
-      - cmd/ui/v1alpha3/*
-      - examples/v1alpha3/*
-      - test/e2e/v1alpha3/*
-      - test/scripts/v1alpha3/*
-      - test/suggestion/v1alpha3/*
-      - test/unit/v1alpha3/*
-      - test/workflows/*
-      - manifests/v1alpha3/*
-      - scripts/v1alpha3/*
-      - vendor/*
-      - prow_config.yaml
-    params:
-      registry: "gcr.io/kubeflow-images-public"
+  # - app_dir: kubeflow/katib/test/workflows
+  #   component: workflows-v1alpha3
+  #   name: e2e-v1alpha3
+  #   job_types:
+  #     - presubmit
+  #   include_dirs:
+  #     - pkg/apis/controller/common/v1alpha3/*
+  #     - pkg/apis/controller/experiments/v1alpha3/*
+  #     - pkg/apis/controller/trials/v1alpha3/*
+  #     - pkg/apis/controller/suggestions/v1alpha3/*
+  #     - pkg/apis/controller/a*.go
+  #     - pkg/apis/manager/health/*
+  #     - pkg/apis/manager/v1alpha3/*
+  #     - pkg/apis/v1alpha3/*
+  #     - pkg/common/v1alpha3/*
+  #     - pkg/controller.v1alpha3/*
+  #     - pkg/db/v1alpha3/*
+  #     - pkg/job/v1alpha3/*
+  #     - pkg/metricscollector/v1alpha3/*
+  #     - pkg/suggestion/v1alpha3/*
+  #     - pkg/ui/v1alpha3/*
+  #     - pkg/util/v1alpha3/*
+  #     - pkg/webhook/v1alpha3/*
+  #     - cmd/db-manager/v1alpha3/*
+  #     - cmd/katib-controller/v1alpha3/*
+  #     - cmd/metricscollector/v1alpha3/*
+  #     - cmd/suggestion/chocolate/v1alpha3/*
+  #     - cmd/suggestion/hyperband/v1alpha3/*
+  #     - cmd/suggestion/hyperopt/v1alpha3/*
+  #     - cmd/suggestion/nas/enas/v1alpha3/*
+  #     - cmd/suggestion/nas/darts/v1alpha3/*
+  #     - cmd/suggestion/skopt/v1alpha3/*
+  #     - cmd/suggestion/goptuna/v1alpha3/*
+  #     - cmd/ui/v1alpha3/*
+  #     - examples/v1alpha3/*
+  #     - test/e2e/v1alpha3/*
+  #     - test/scripts/v1alpha3/*
+  #     - test/suggestion/v1alpha3/*
+  #     - test/unit/v1alpha3/*
+  #     - test/workflows/*
+  #     - manifests/v1alpha3/*
+  #     - scripts/v1alpha3/*
+  #     - vendor/*
+  #     - prow_config.yaml
+  #   params:
+  #     registry: "gcr.io/automl-ci"
+  # - app_dir: kubeflow/katib/test/workflows
+  #   component: workflows-v1alpha3
+  #   name: e2e-v1alpha3
+  #   job_types:
+  #     - postsubmit
+  #   include_dirs:
+  #     - pkg/apis/controller/common/v1alpha3/*
+  #     - pkg/apis/controller/experiments/v1alpha3/*
+  #     - pkg/apis/controller/trials/v1alpha3/*
+  #     - pkg/apis/controller/suggestions/v1alpha3/*
+  #     - pkg/apis/controller/a*.go
+  #     - pkg/apis/manager/health/*
+  #     - pkg/apis/manager/v1alpha3/*
+  #     - pkg/apis/v1alpha3/*
+  #     - pkg/common/v1alpha3/*
+  #     - pkg/controller.v1alpha3/*
+  #     - pkg/db/v1alpha3/*
+  #     - pkg/job/v1alpha3/*
+  #     - pkg/metricscollector/v1alpha3/*
+  #     - pkg/suggestion/v1alpha3/*
+  #     - pkg/ui/v1alpha3/*
+  #     - pkg/util/v1alpha3/*
+  #     - pkg/webhook/v1alpha3/*
+  #     - cmd/db-manager/v1alpha3/*
+  #     - cmd/katib-controller/v1alpha3/*
+  #     - cmd/metricscollector/v1alpha3/*
+  #     - cmd/suggestion/chocolate/v1alpha3/*
+  #     - cmd/suggestion/hyperband/v1alpha3/*
+  #     - cmd/suggestion/hyperopt/v1alpha3/*
+  #     - cmd/suggestion/nas/enas/v1alpha3/*
+  #     - cmd/suggestion/nas/darts/v1alpha3/*
+  #     - cmd/suggestion/skopt/v1alpha3/*
+  #     - cmd/suggestion/goptuna/v1alpha3/*
+  #     - cmd/ui/v1alpha3/*
+  #     - examples/v1alpha3/*
+  #     - test/e2e/v1alpha3/*
+  #     - test/scripts/v1alpha3/*
+  #     - test/suggestion/v1alpha3/*
+  #     - test/unit/v1alpha3/*
+  #     - test/workflows/*
+  #     - manifests/v1alpha3/*
+  #     - scripts/v1alpha3/*
+  #     - vendor/*
+  #     - prow_config.yaml
+  #   params:
+  #     registry: "gcr.io/kubeflow-images-public"
   - app_dir: kubeflow/katib/test/workflows
     component: workflows-v1beta1
     name: e2e-v1beta1
@@ -138,50 +138,50 @@ workflows:
       - vendor/*
       - prow_config.yaml
     params:
-      registry: "gcr.io/automl-ci"
-  - app_dir: kubeflow/katib/test/workflows
-    component: workflows-v1beta1
-    name: e2e-v1beta1
-    job_types:
-      - postsubmit
-    include_dirs:
-      - pkg/apis/controller/common/v1beta1/*
-      - pkg/apis/controller/experiments/v1beta1/*
-      - pkg/apis/controller/trials/v1beta1/*
-      - pkg/apis/controller/suggestions/v1beta1/*
-      - pkg/apis/controller/a*.go
-      - pkg/apis/manager/health/*
-      - pkg/apis/manager/v1beta1/*
-      - pkg/apis/v1beta1/*
-      - pkg/common/v1beta1/*
-      - pkg/controller.v1beta1/*
-      - pkg/db/v1beta1/*
-      - pkg/job/v1beta1/*
-      - pkg/metricscollector/v1beta1/*
-      - pkg/suggestion/v1beta1/*
-      - pkg/ui/v1beta1/*
-      - pkg/util/v1beta1/*
-      - pkg/webhook/v1beta1/*
-      - cmd/db-manager/v1beta1/*
-      - cmd/katib-controller/v1beta1/*
-      - cmd/metricscollector/v1beta1/*
-      - cmd/suggestion/chocolate/v1beta1/*
-      - cmd/suggestion/hyperband/v1beta1/*
-      - cmd/suggestion/hyperopt/v1beta1/*
-      - cmd/suggestion/nas/enas/v1beta1/*
-      - cmd/suggestion/nas/darts/v1beta1/*
-      - cmd/suggestion/skopt/v1beta1/*
-      - cmd/suggestion/goptuna/v1beta1/*
-      - cmd/ui/v1beta1/*
-      - examples/v1beta1/*
-      - test/e2e/v1beta1/*
-      - test/scripts/v1beta1/*
-      - test/suggestion/v1beta1/*
-      - test/unit/v1beta1/*
-      - test/workflows/*
-      - manifests/v1beta1/*
-      - scripts/v1beta1/*
-      - vendor/*
-      - prow_config.yaml
-    params:
-      registry: "gcr.io/kubeflow-images-public"
+      registry: http://527798164940.dkr.ecr.us-west-2.amazonaws.com/
+  # - app_dir: kubeflow/katib/test/workflows
+  #   component: workflows-v1beta1
+  #   name: e2e-v1beta1
+  #   job_types:
+  #     - postsubmit
+  #   include_dirs:
+  #     - pkg/apis/controller/common/v1beta1/*
+  #     - pkg/apis/controller/experiments/v1beta1/*
+  #     - pkg/apis/controller/trials/v1beta1/*
+  #     - pkg/apis/controller/suggestions/v1beta1/*
+  #     - pkg/apis/controller/a*.go
+  #     - pkg/apis/manager/health/*
+  #     - pkg/apis/manager/v1beta1/*
+  #     - pkg/apis/v1beta1/*
+  #     - pkg/common/v1beta1/*
+  #     - pkg/controller.v1beta1/*
+  #     - pkg/db/v1beta1/*
+  #     - pkg/job/v1beta1/*
+  #     - pkg/metricscollector/v1beta1/*
+  #     - pkg/suggestion/v1beta1/*
+  #     - pkg/ui/v1beta1/*
+  #     - pkg/util/v1beta1/*
+  #     - pkg/webhook/v1beta1/*
+  #     - cmd/db-manager/v1beta1/*
+  #     - cmd/katib-controller/v1beta1/*
+  #     - cmd/metricscollector/v1beta1/*
+  #     - cmd/suggestion/chocolate/v1beta1/*
+  #     - cmd/suggestion/hyperband/v1beta1/*
+  #     - cmd/suggestion/hyperopt/v1beta1/*
+  #     - cmd/suggestion/nas/enas/v1beta1/*
+  #     - cmd/suggestion/nas/darts/v1beta1/*
+  #     - cmd/suggestion/skopt/v1beta1/*
+  #     - cmd/suggestion/goptuna/v1beta1/*
+  #     - cmd/ui/v1beta1/*
+  #     - examples/v1beta1/*
+  #     - test/e2e/v1beta1/*
+  #     - test/scripts/v1beta1/*
+  #     - test/suggestion/v1beta1/*
+  #     - test/unit/v1beta1/*
+  #     - test/workflows/*
+  #     - manifests/v1beta1/*
+  #     - scripts/v1beta1/*
+  #     - vendor/*
+  #     - prow_config.yaml
+  #   params:
+  #     registry: "gcr.io/kubeflow-images-public"

test/e2e/v1beta1/run-e2e-experiment.go

@@ -7,7 +7,9 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
+	"os/exec"
 	"strconv"
+	"strings"
 	"time"
 
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -52,19 +54,25 @@ func main() {
 		log.Fatal("Experiment name is missing")
 	}
 	expName := os.Args[1]
-	b, err := ioutil.ReadFile(expName)
+	byteExp, err := ioutil.ReadFile(expName)
 	if err != nil {
 		log.Fatal("Error in reading file ", err)
 	}
+
+	// Replace batch size to number of epochs for faster execution.
+	strExp := strings.Replace(string(byteExp), "--batch-size=64", "--num-epochs=2", -1)
+
 	exp := &experimentsv1beta1.Experiment{}
-	buf := bytes.NewBufferString(string(b))
+	buf := bytes.NewBufferString(strExp)
 	if err = k8syaml.NewYAMLOrJSONDecoder(buf, 1024).Decode(exp); err != nil {
 		log.Fatal("Yaml decode error ", err)
 	}
+
 	kclient, err := katibclient.NewClient(client.Options{})
 	if err != nil {
 		log.Fatal("NewClient for Katib failed: ", err)
 	}
+
 	if exp.Spec.Algorithm.AlgorithmName != "hyperband" && exp.Spec.Algorithm.AlgorithmName != "darts" {
 		// Hyperband will validate the parallel trial count,
 		// thus we should not change it.
@@ -86,7 +94,7 @@ func main() {
 		}
 		log.Printf("Waiting for Experiment %s to finish.", exp.Name)
 		log.Printf(`Experiment %s's trials: %d trials, %d pending trials,
-%d running trials, %d killed trials, %d succeeded trials, %d failed trials.`,
+	%d running trials, %d killed trials, %d succeeded trials, %d failed trials.`,
 			exp.Name,
 			exp.Status.Trials, exp.Status.TrialsPending, exp.Status.TrialsRunning,
 			exp.Status.TrialsKilled, exp.Status.TrialsSucceeded, exp.Status.TrialsFailed)
@@ -187,5 +195,24 @@ func main() {
 		}
 	}
 
-	log.Printf("Experiment has recorded best current Optimal Trial %v", exp.Status.CurrentOptimalTrial)
+	log.Printf("Experiment has recorded best current Optimal Trial %v\n", exp.Status.CurrentOptimalTrial)
+
+	out, err := exec.Command("kubectl", "describe", "suggestion", exp.Name, "-n", exp.Namespace).Output()
+	if err != nil {
+		log.Fatalf("Execute \"kubectl describe suggestion\" failed: %v", err)
+	}
+	fmt.Println(string(out))
+
+	out, err = exec.Command("kubectl", "describe", "experiment", exp.Name, "-n", exp.Namespace).Output()
+	if err != nil {
+		log.Fatalf("Execute \"kubectl describe experiment\" failed: %v", err)
+	}
+	fmt.Println(string(out))
+
+	log.Printf("Deleting Experiment: %v\n", exp.Name)
+	err = kclient.DeleteRuntimeObject(exp)
+	if err != nil {
+		log.Fatalf("Unable to delete Experiment: %v, error: %v", exp.Name, err)
+	}
+
 }