Switch to AWS CI/CD (#1356)

* Add changes for AWS test infra

* Remove comment from resume e2e

* Change worker image

* Refactor e2e test script

* Replace create and delete cluster with testing scripts

* Fix cluster name

* Fix delete cluster

* Test without folder for GOPATH

* Add AWS creds to env

* Comment creds

* Delete v1alpha3 workflow from prow
Add ECR env

* Add AWS cred

* Get other build for all images

* Attach volume to create and delete cluster

* Fix path for NAS suggestions

* Change make deploy

* Fix path

* Change deploy

* Move create cluster to build

* Fix path to valid exp

* Remove

* Change command

* Fix region

* Fix run e2e go path

* Add backoff

* Add github.com to folder

* Add github to src folder

* Fix Katib path

* Print CRDs in e2e Experiment

* Show known types

* Trigger CI

* Set TypeMeta for experiment

* Print known types

* Build binary e2e

* manually create experiment

* Print ns list

* Remove GCP auth

* Set kube config

* Add other e2e tests

* Remove bin

* Fix template name

* Return exp in case of error

* Deploy TF and PyTorch controllers

* Create Kubeflow namespace

* Remove v1alpha3 tests

* Remove Katib client changes

* Add ttl seconds after finished

* Change to 5 hours

* Increase activeDeadlineSeconds

* Add comments

* Add release workflow

prow_config.yaml

@@ -1,98 +1,6 @@
 # This file configures the workflows to trigger in our Prow jobs.
 # see kubeflow/testing/py/run_e2e_workflow.py
 workflows:
-  - app_dir: kubeflow/katib/test/workflows
-    component: workflows-v1alpha3
-    name: e2e-v1alpha3
-    job_types:
-      - presubmit
-    include_dirs:
-      - pkg/apis/controller/common/v1alpha3/*
-      - pkg/apis/controller/experiments/v1alpha3/*
-      - pkg/apis/controller/trials/v1alpha3/*
-      - pkg/apis/controller/suggestions/v1alpha3/*
-      - pkg/apis/controller/a*.go
-      - pkg/apis/manager/health/*
-      - pkg/apis/manager/v1alpha3/*
-      - pkg/apis/v1alpha3/*
-      - pkg/common/v1alpha3/*
-      - pkg/controller.v1alpha3/*
-      - pkg/db/v1alpha3/*
-      - pkg/job/v1alpha3/*
-      - pkg/metricscollector/v1alpha3/*
-      - pkg/suggestion/v1alpha3/*
-      - pkg/ui/v1alpha3/*
-      - pkg/util/v1alpha3/*
-      - pkg/webhook/v1alpha3/*
-      - cmd/db-manager/v1alpha3/*
-      - cmd/katib-controller/v1alpha3/*
-      - cmd/metricscollector/v1alpha3/*
-      - cmd/suggestion/chocolate/v1alpha3/*
-      - cmd/suggestion/hyperband/v1alpha3/*
-      - cmd/suggestion/hyperopt/v1alpha3/*
-      - cmd/suggestion/nas/enas/v1alpha3/*
-      - cmd/suggestion/nas/darts/v1alpha3/*
-      - cmd/suggestion/skopt/v1alpha3/*
-      - cmd/suggestion/goptuna/v1alpha3/*
-      - cmd/ui/v1alpha3/*
-      - examples/v1alpha3/*
-      - test/e2e/v1alpha3/*
-      - test/scripts/v1alpha3/*
-      - test/suggestion/v1alpha3/*
-      - test/unit/v1alpha3/*
-      - test/workflows/*
-      - manifests/v1alpha3/*
-      - scripts/v1alpha3/*
-      - vendor/*
-      - prow_config.yaml
-    params:
-      registry: "gcr.io/automl-ci"
-  - app_dir: kubeflow/katib/test/workflows
-    component: workflows-v1alpha3
-    name: e2e-v1alpha3
-    job_types:
-      - postsubmit
-    include_dirs:
-      - pkg/apis/controller/common/v1alpha3/*
-      - pkg/apis/controller/experiments/v1alpha3/*
-      - pkg/apis/controller/trials/v1alpha3/*
-      - pkg/apis/controller/suggestions/v1alpha3/*
-      - pkg/apis/controller/a*.go
-      - pkg/apis/manager/health/*
-      - pkg/apis/manager/v1alpha3/*
-      - pkg/apis/v1alpha3/*
-      - pkg/common/v1alpha3/*
-      - pkg/controller.v1alpha3/*
-      - pkg/db/v1alpha3/*
-      - pkg/job/v1alpha3/*
-      - pkg/metricscollector/v1alpha3/*
-      - pkg/suggestion/v1alpha3/*
-      - pkg/ui/v1alpha3/*
-      - pkg/util/v1alpha3/*
-      - pkg/webhook/v1alpha3/*
-      - cmd/db-manager/v1alpha3/*
-      - cmd/katib-controller/v1alpha3/*
-      - cmd/metricscollector/v1alpha3/*
-      - cmd/suggestion/chocolate/v1alpha3/*
-      - cmd/suggestion/hyperband/v1alpha3/*
-      - cmd/suggestion/hyperopt/v1alpha3/*
-      - cmd/suggestion/nas/enas/v1alpha3/*
-      - cmd/suggestion/nas/darts/v1alpha3/*
-      - cmd/suggestion/skopt/v1alpha3/*
-      - cmd/suggestion/goptuna/v1alpha3/*
-      - cmd/ui/v1alpha3/*
-      - examples/v1alpha3/*
-      - test/e2e/v1alpha3/*
-      - test/scripts/v1alpha3/*
-      - test/suggestion/v1alpha3/*
-      - test/unit/v1alpha3/*
-      - test/workflows/*
-      - manifests/v1alpha3/*
-      - scripts/v1alpha3/*
-      - vendor/*
-      - prow_config.yaml
-    params:
-      registry: "gcr.io/kubeflow-images-public"
   - app_dir: kubeflow/katib/test/workflows
     component: workflows-v1beta1
     name: e2e-v1beta1
@@ -138,9 +46,10 @@ workflows:
       - vendor/*
       - prow_config.yaml
     params:
-      registry: "gcr.io/automl-ci"
+      registry: 527798164940.dkr.ecr.us-west-2.amazonaws.com
+  # TODO (andreyvelich): Temporary workflow to release Katib images to kubeflow-images-public registry.
   - app_dir: kubeflow/katib/test/workflows
-    component: workflows-v1beta1
+    component: workflows-v1beta1-release
     name: e2e-v1beta1
     job_types:
       - postsubmit